--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:08:17.648698Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-86dc6-9854ff79c-rrpx7 to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:08:17Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be46928826b87f namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: d8440b8e-87ba-4dc7-a07b-420605827d9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:08:18Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:08:18Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be4692a1ca5b62 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 1109af24-737c-4ee8-ad9a-36aa53c1ea22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-86dc6} kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:08:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:18Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be4692a2fada02 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 43433ec0-4320-4709-b520-ffb371c5cfcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-86dc6} kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:08:18Z" message: 'Created container: ensemble-graph-86dc6' metadata: creationTimestamp: "2026-07-01T21:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:18Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be4692a60e6387 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: da25d60b-7039-4e14-9899-301b0650e92a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-86dc6} kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:08:18Z" message: Started container ensemble-graph-86dc6 metadata: creationTimestamp: "2026-07-01T21:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:18Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be4692a686462c namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 70c42dc8-b5f3-446b-a597-7a320d0ce9da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-86dc6} kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Stopping container ensemble-graph-86dc6 metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be4705b97b9061 namespace: kserve-ci-e2e-test resourceVersion: "25309" uid: e07b42a0-620f-4a83-aea5-e8610600714e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-86dc6} kind: Pod name: ensemble-graph-86dc6-9854ff79c-rrpx7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: a95fe24d-64da-4e14-9b30-c39ee5ba8f12 kind: Event lastTimestamp: "2026-07-01T21:17:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:00Z" name: ensemble-graph-86dc6-9854ff79c-rrpx7.18be47065e5f24b6 namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: a594c4fd-41bd-4d3c-95e8-6751dbf35966 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-86dc6-9854ff79c namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 80754a61-98e1-4b67-ad3a-d41c74c76c29 kind: Event lastTimestamp: "2026-07-01T21:08:17Z" message: 'Created pod: ensemble-graph-86dc6-9854ff79c-rrpx7' metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:17Z" name: ensemble-graph-86dc6-9854ff79c.18be469287b66948 namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: 53af190c-59d6-409d-b9f3-da092a74eed2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 81df214d-2dee-402d-a12a-6a4041b8e6d6 kind: Event lastTimestamp: "2026-07-01T21:08:17Z" message: Scaled up replica set ensemble-graph-86dc6-9854ff79c from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:17Z" name: ensemble-graph-86dc6.18be469286c14ff4 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 6d9fe4f9-475b-4a73-9539-214c6d918828 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: ccdf106d-8aa2-4543-9c59-87b263800f51 kind: Event lastTimestamp: "2026-07-01T21:08:17Z" message: InferenceGraph [ensemble-graph-86dc6] is Ready metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:17Z" name: ensemble-graph-86dc6.18be46928b9390cb namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 5ec5c77f-21c8-4767-9ee2-32b85e5e1f8f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 5b9ef2aa-da21-43b7-97ce-824a4d12c034 kind: Event lastTimestamp: "2026-07-01T21:09:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:02Z" name: ensemble-graph-86dc6.18be469605333cf1 namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: 2fcbd710-ceb7-4bc5-93eb-709a8ee1cd05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 5b9ef2aa-da21-43b7-97ce-824a4d12c034 kind: Event lastTimestamp: "2026-07-01T21:09:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:02Z" name: ensemble-graph-86dc6.18be469605725f91 namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 0e4ad446-cad9-4db6-a3df-0d38bd42be0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:07.371445Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-f1d61-6f568488db-8x8cf to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be46822b4cfb87 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 74be60c0-eb36-4a97-8ec3-42fb08de7dfe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-f1d61-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be46823a1b21cf namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 59a4f9e7-23ea-402d-b055-007a12be2baa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:08Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:08Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:08Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be468268f8c012 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: cbb4036d-eb81-422b-bdf6-ee4d9aca0894 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f1d61} kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:08Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be46826a3015c3 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 14a6de73-4210-449c-a65c-4a0abb1f7d04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f1d61} kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:08Z" message: 'Created container: ensemble-graph-f1d61' metadata: creationTimestamp: "2026-07-01T21:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:08Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be46826dc70696 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 1ee690e6-edcb-429c-9a53-e994d996a7b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f1d61} kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:08Z" message: Started container ensemble-graph-f1d61 metadata: creationTimestamp: "2026-07-01T21:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:08Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be46826e42d425 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2b3df455-8038-4155-b965-b45daf1af3e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f1d61} kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Stopping container ensemble-graph-f1d61 metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be4684837b2894 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: c2079c8d-20bb-4c3d-9c59-744760b96e98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f1d61} kind: Pod name: ensemble-graph-f1d61-6f568488db-8x8cf namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d8d93fb8-5a12-4e49-9b36-83a84dd0865d kind: Event lastTimestamp: "2026-07-01T21:07:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:44Z" name: ensemble-graph-f1d61-6f568488db-8x8cf.18be4685136b5d76 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 45e5473e-4f36-4df2-9a29-0936f069577a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-f1d61-6f568488db namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 1cc94bfe-740a-4e0d-ad19-e1d682bf1c6f kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: 'Created pod: ensemble-graph-f1d61-6f568488db-8x8cf' metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61-6f568488db.18be46822ac41e42 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: d4188928-d424-4c13-a8ba-17f436300fd2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 397c0ea6-91bc-4a9b-a165-0655f6094479 kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: Scaled up replica set ensemble-graph-f1d61-6f568488db from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61.18be468229a5de35 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: a49f018e-b40a-43cc-a407-93e3235e4f8c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 93a68b5a-39ac-4b6a-976c-6c6cec18026c kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: InferenceGraph [ensemble-graph-f1d61] is Ready metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61.18be46822d42af8c namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 864900c7-cb43-4a66-bdc0-97b145b17091 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 93a68b5a-39ac-4b6a-976c-6c6cec18026c kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-f1d61": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61.18be46822f511ee1 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: f5d892a3-46c3-4700-80df-7bd37767ea0e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 93a68b5a-39ac-4b6a-976c-6c6cec18026c kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:07Z" name: ensemble-graph-f1d61.18be46822f517ff1 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 7f8dc2df-5b34-48ee-a0d3-57017ace0097 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:54:32.419123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:54:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45d264a2c490 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 16510318-be27-4cb9-9986-6e7a95d09bd4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:54:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45d27dd79c44 namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 584920f5-cdd7-477f-90c8-2bb75fb1c93e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45d27f2484da namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 7812b005-33ca-40b9-8d0f-45a67d9a028b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:08Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" in 36.126s (36.126s including waiting). Image size: 1334629466 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:08Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45dae86f3642 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 582c52ef-3e16-44bf-8891-9602cf9441a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45daef79f599 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 505ba269-aea4-4cfa-a794-3520b46e63ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45daf06c2011 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 118e7cb2-de68-419a-81b5-fd7ca3b029d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45daf079701d namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 7b0fd9cb-a658-4ad8-b301-5ea62229c350 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.53s (4.53s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45dbfe7d79c3 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: cb4591f6-5363-4a61-a161-ce886cdff14e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45dc0168ea1d namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: e7999447-3b64-4138-862f-be36bfe619ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45dc01e18b6c namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 2247fb92-de62-43b3-93d9-64b3fdb1adcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:55:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:51Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45dc544d1d42 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: ce51c554-3c76-46ac-84aa-74e05eb88730 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45ee596babd7 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 16c70eae-1094-4794-ad2a-1e2cdbea78f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45ee596c2dc1 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: a3f217ed-2548-4fc4-8513-8ea57f4e1685 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1a485aa7-57db-42b5-98c1-9f180e964632 kind: Event lastTimestamp: "2026-07-01T20:56:36Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:36Z" name: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc.18be45ef309212c7 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 157a8848-afe4-4a9c-976e-ac6213da9649 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-25c51-predictor-64f4bf5df6 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: 489ec02b-de27-4b36-83bd-b66a857f7b74 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'Created pod: error-404-isvc-25c51-predictor-64f4bf5df6-t9jxc' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: error-404-isvc-25c51-predictor-64f4bf5df6.18be45d264443729 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: dbac83e7-617c-4792-b7c7-6cc912d9191b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 5834e599-349e-41c4-8129-34dafea2de3a kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Scaled up replica set error-404-isvc-25c51-predictor-64f4bf5df6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: error-404-isvc-25c51-predictor.18be45d2612fffb6 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: f6ad24f5-e902-4d51-b1d8-5ed332fad36d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 02b94a8e-86d1-4ea6-88a7-7c8fc6274e33 kind: Event lastTimestamp: "2026-07-01T20:55:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:47Z" name: error-404-isvc-25c51-predictor.18be45d5dfc7b639 namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 6d859df0-b384-418f-9937-463ef6affa96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 02b94a8e-86d1-4ea6-88a7-7c8fc6274e33 kind: Event lastTimestamp: "2026-07-01T20:55:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:47Z" name: error-404-isvc-25c51-predictor.18be45d5e010107a namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 09bfceda-a456-4b42-b189-6bcddba9c099 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 02b94a8e-86d1-4ea6-88a7-7c8fc6274e33 kind: Event lastTimestamp: "2026-07-01T20:56:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:17Z" name: error-404-isvc-25c51-predictor.18be45e05c2b0b14 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: d62e08e5-1255-48d3-8af3-3739718354e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 02b94a8e-86d1-4ea6-88a7-7c8fc6274e33 kind: Event lastTimestamp: "2026-07-01T20:56:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:17Z" name: error-404-isvc-25c51-predictor.18be45e05c71f8f3 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: fed7f933-4989-4692-b74f-b02fa1512dc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 02b94a8e-86d1-4ea6-88a7-7c8fc6274e33 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: horizontalpodautoscalers.autoscaling "error-404-isvc-25c51-predictor" not found metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-25c51-predictor.18be45ee58ba71e2 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: e88dc238-633b-4905-9ef3-65ef1dca792f reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c51 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 3bbdd386-e157-4313-a161-525d947dcbb8 kind: Event lastTimestamp: "2026-07-01T20:56:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:01Z" name: error-404-isvc-25c51.18be45d2614070b6 namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 6ba1d8c5-4e46-4f8f-be34-5b96a06b72f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c51 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 3bbdd386-e157-4313-a161-525d947dcbb8 kind: Event lastTimestamp: "2026-07-01T20:56:01Z" message: InferenceService [error-404-isvc-25c51] is Ready metadata: creationTimestamp: "2026-07-01T20:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:01Z" name: error-404-isvc-25c51.18be45e70fb81098 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 7aefb96b-d57c-414d-b226-38d187fd599a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:16:32.788161Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25374" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:16:32Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705d0c37307 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 1a5337fa-9978-4b02-9c7a-ce83102b5e11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25393" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705ea36a047 namespace: kserve-ci-e2e-test resourceVersion: "25412" uid: a8a729ff-6c5d-4ff5-92be-da9edad7b722 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705ebaf5bab namespace: kserve-ci-e2e-test resourceVersion: "25414" uid: 8af88834-02e9-4ce9-81a2-b93df502ed9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705eeb1623b namespace: kserve-ci-e2e-test resourceVersion: "25417" uid: bf988b1b-db42-4189-99d0-09c0379626d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705ef1f4c92 namespace: kserve-ci-e2e-test resourceVersion: "25418" uid: a92b382b-4e0f-434a-926e-ef335cdba1ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705ef29a629 namespace: kserve-ci-e2e-test resourceVersion: "25419" uid: 37674e8b-7707-42a4-bdc9-182d21c50e9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705f2959ede namespace: kserve-ci-e2e-test resourceVersion: "25420" uid: 314e6521-4c4f-416e-b944-7319e2cb35cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be4705f302fe9b namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: c9223668-5f4c-45e7-b10e-9a72c0a54c32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:17:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:10Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be470628ea1fb4 namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 4db2ac8c-7732-4683-8b0c-b0c802d39375 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be47187f618d67 namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 06e4b800-6723-47cb-b030-0140d78f6b37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be47187f61e756 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: a9fe2eae-c12f-4870-b153-2c5533c1da56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: facd928f-2243-4d2e-addf-d9c156d21db4 kind: Event lastTimestamp: "2026-07-01T21:17:55Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:55Z" name: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw.18be471905326340 namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: afdf3d01-b33f-4709-99c0-3a558f7175ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3b58c-predictor-5cc76b4998 namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: dcc52310-6466-40c8-a939-c74a885dabbe kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: 'Created pod: error-404-isvc-3b58c-predictor-5cc76b4998-dnlrw' metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:32Z" name: error-404-isvc-3b58c-predictor-5cc76b4998.18be4705d04e7921 namespace: kserve-ci-e2e-test resourceVersion: "25375" uid: c480d2da-d227-4946-8b3f-d7b572f0ba9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 4e57a35f-9732-4c78-9f10-8df6f80c0e6b kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Scaled up replica set error-404-isvc-3b58c-predictor-5cc76b4998 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:32Z" name: error-404-isvc-3b58c-predictor.18be4705ce0472dc namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: cf9a89fe-2fb6-4c99-8547-dcea6665a678 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 6ff20e26-6ba6-4412-8ef0-890bbafbc207 kind: Event lastTimestamp: "2026-07-01T21:16:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:47Z" name: error-404-isvc-3b58c-predictor.18be47094c7c4a8a namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: e512bd15-dafc-404d-b47c-fe6f372509f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 6ff20e26-6ba6-4412-8ef0-890bbafbc207 kind: Event lastTimestamp: "2026-07-01T21:16:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:47Z" name: error-404-isvc-3b58c-predictor.18be47094ce680c7 namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: bbafa5e5-c8a9-49b5-b63a-f89a4a0714c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:17:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 6ff20e26-6ba6-4412-8ef0-890bbafbc207 kind: Event lastTimestamp: "2026-07-01T21:17:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:47Z" name: error-404-isvc-3b58c-predictor.18be470ccba79a89 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: fc2c8b9b-2b13-46dd-bd95-ca3c1b45f73b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:17:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 6ff20e26-6ba6-4412-8ef0-890bbafbc207 kind: Event lastTimestamp: "2026-07-01T21:17:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:47Z" name: error-404-isvc-3b58c-predictor.18be470ccc294664 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 7aa505e6-5bc8-4d8c-bb95-b56c4b364edc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3b58c namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 0d40eb44-419b-440b-bada-9236a8895ac9 kind: Event lastTimestamp: "2026-07-01T21:17:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:20Z" name: error-404-isvc-3b58c.18be4705ce05b62c namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: a10b0c6c-132b-4496-a049-164075d1e47e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 0d40eb44-419b-440b-bada-9236a8895ac9 kind: Event lastTimestamp: "2026-07-01T21:17:20Z" message: InferenceService [error-404-isvc-3b58c] is Ready metadata: creationTimestamp: "2026-07-01T21:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:20Z" name: error-404-isvc-3b58c.18be4710e49e3c48 namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 7d49208a-1d9d-410e-91fe-cea11f7a1a8c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:17:52.976080Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:17:52Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be47187c55fad6 namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: 3a09b32a-7529-4a3b-93db-5a1e65615afb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6396f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718849981c9 namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: c8e2416e-f613-4727-8ca2-eefd001d9e66 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718b9e547bd namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: 06c1ac73-cfc3-45ee-b17d-5cd7d60f1e37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718bba6e5c9 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: 5588b087-8262-4cdc-912a-68b4cc5f6fe9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718bf795bc4 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: eb4a3459-cc43-4323-a1bc-601fe8f57aa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718bfeeb66d namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 3adfa39b-d369-4ecb-a303-7c0f3b2c911c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718bff85762 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: 65c2eae9-2c79-4425-b8fd-f1f848612352 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718c281641b namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: e358f262-656d-42c1-8792-332efe1f83ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718c2edb8ae namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 7fc0d423-71d4-406e-b69d-15601ee5df43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:27:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:10Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4718dedcdc11 namespace: kserve-ci-e2e-test resourceVersion: "30814" uid: 088682bc-a066-4ee6-8570-dac1b669a60f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:27:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:07Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4799af51e99f namespace: kserve-ci-e2e-test resourceVersion: "30789" uid: 1ab331b7-3d3c-4287-9b6c-8a5c8512d738 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:27:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:07Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be4799af525d81 namespace: kserve-ci-e2e-test resourceVersion: "30791" uid: 14fbd4c7-54c3-4db5-865e-b26b107a4b6e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: a87761b4-21ed-4470-ae67-77e1c811ee2f kind: Event lastTimestamp: "2026-07-01T21:27:10Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:10Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f.18be479a533c2f47 namespace: kserve-ci-e2e-test resourceVersion: "30813" uid: e0d9cfb3-7044-4763-8ae5-e8ece1dc7a65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6396f-predictor-77fbbbfd5b namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 53224c81-dd3a-48bf-8bf8-08ad2287c9ca kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'Created pod: error-404-isvc-6396f-predictor-77fbbbfd5b-g4m9f' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:52Z" name: error-404-isvc-6396f-predictor-77fbbbfd5b.18be47187bef0000 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 5c9ed0c0-df88-4ba2-8d6e-772754b818eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: e43707ad-95d0-4e3b-9ff0-202756981a38 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: Scaled up replica set error-404-isvc-6396f-predictor-77fbbbfd5b from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:52Z" name: error-404-isvc-6396f-predictor.18be47187a3feaf2 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 619adf10-4923-47ad-911c-4e27dae31ef8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: bff81f89-b297-4a7a-ac8f-185522e7ee05 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: error-404-isvc-6396f-predictor.18be471bf8cdf217 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: 1eed3b52-a0ee-4854-bbc4-10b1610f0f40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: bff81f89-b297-4a7a-ac8f-185522e7ee05 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: error-404-isvc-6396f-predictor.18be471bf935e3ba namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 3a201b7d-cb00-45d5-8fe5-d8065bc27aad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: bff81f89-b297-4a7a-ac8f-185522e7ee05 kind: Event lastTimestamp: "2026-07-01T21:19:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:08Z" name: error-404-isvc-6396f-predictor.18be4722f6b1b7c8 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: f8fc4656-705a-4c51-9767-ee80397180a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: bff81f89-b297-4a7a-ac8f-185522e7ee05 kind: Event lastTimestamp: "2026-07-01T21:19:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:08Z" name: error-404-isvc-6396f-predictor.18be4722f7087432 namespace: kserve-ci-e2e-test resourceVersion: "27420" uid: e6cc4b05-51e3-4789-be30-d51e4f14965b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: e8caa535-9b83-4111-aa14-15edced0ad9f kind: Event lastTimestamp: "2026-07-01T21:18:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:40Z" name: error-404-isvc-6396f.18be47187a4a53c8 namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 93a89859-20e9-4d48-8c71-15dbe973addd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: e8caa535-9b83-4111-aa14-15edced0ad9f kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-6396f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6396f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:52Z" name: error-404-isvc-6396f.18be47187ac83107 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 6a90229e-f0d2-4079-9348-50ed02d0cc4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: e8caa535-9b83-4111-aa14-15edced0ad9f kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6396f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:52Z" name: error-404-isvc-6396f.18be47187ac867c4 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: 57a2a666-9c93-4477-abb8-0b2b6f27f937 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: e8caa535-9b83-4111-aa14-15edced0ad9f kind: Event lastTimestamp: "2026-07-01T21:18:40Z" message: InferenceService [error-404-isvc-6396f] is Ready metadata: creationTimestamp: "2026-07-01T21:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:40Z" name: error-404-isvc-6396f.18be47239ada7ed4 namespace: kserve-ci-e2e-test resourceVersion: "27176" uid: 8c35eedb-3013-4667-a789-f6d7e90f5c64 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:06:17.304092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7cf3f-predictor-575f8587c8-67j58 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676830dddaf namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: d014762e-612e-4a93-9ac6-6664a983488e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be46769d65b904 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 6f095ed5-a7c7-474d-9e90-8e8c41f834fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be46769ebb6d38 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: b220adc2-65ca-4c3f-b6d2-0e2cb02101c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676a1f1d5d4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 68684fa4-b735-456d-983e-79ad36080418 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676a28071bc namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: d5dbc24f-f01a-4723-8778-d988e4a993a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676a28b1e34 namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 9006e3d0-3c23-4cdc-9ff9-0510660ed6d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676a51746eb namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 28aeba5b-3e6c-4367-8102-cd3e3d41075e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4676a586a95f namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: beee57b4-895c-494a-8170-1386151f5a0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:06:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:06:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:55Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4677184a738f namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: fb1033ff-59a5-4348-b6e4-4b7009c29c6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4689280f2702 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 9411b3db-d653-4f52-bbbd-b32fa1388339 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be468928108c80 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 43143477-4d98-40a7-9cf0-74f6b1e1ea4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 6993df91-d98d-4a74-ba21-84fa315d343e kind: Event lastTimestamp: "2026-07-01T21:07:40Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:40Z" name: error-404-isvc-7cf3f-predictor-575f8587c8-67j58.18be4689f4f6a913 namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 9a41da10-4aa4-4c64-b70f-e7133d8f084b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7cf3f-predictor-575f8587c8 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 8d8b1feb-cb3b-4e7b-9094-1f32174e5931 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created pod: error-404-isvc-7cf3f-predictor-575f8587c8-67j58' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor-575f8587c8.18be4676827fcd8e namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: aab6a2c0-7bea-445b-885c-3fb862057c8c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 77776d5c-9401-4764-9f14-487ceffda9ee kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Scaled up replica set error-404-isvc-7cf3f-predictor-575f8587c8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f-predictor.18be467680533dde namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 4d244dbf-d372-42e0-a343-7816bf5eb4ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: d3a2c3de-9225-40f2-ad19-43422b8dbe5b kind: Event lastTimestamp: "2026-07-01T21:07:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:02Z" name: error-404-isvc-7cf3f-predictor.18be4679fecf2be1 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 2a2c738e-04ae-4978-823e-c85f0b902434 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: d3a2c3de-9225-40f2-ad19-43422b8dbe5b kind: Event lastTimestamp: "2026-07-01T21:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:02Z" name: error-404-isvc-7cf3f-predictor.18be4679ff2de6a2 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 50c459ab-34d5-41b0-9688-635e46642616 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: d3a2c3de-9225-40f2-ad19-43422b8dbe5b kind: Event lastTimestamp: "2026-07-01T21:07:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:32Z" name: error-404-isvc-7cf3f-predictor.18be467d7dfe72b4 namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 0f34aa11-e3aa-43ba-bbee-595db46b4e4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: d3a2c3de-9225-40f2-ad19-43422b8dbe5b kind: Event lastTimestamp: "2026-07-01T21:07:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:32Z" name: error-404-isvc-7cf3f-predictor.18be467d7e7245c0 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 6d1a72eb-e46d-4b73-8659-703532701082 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 85747e84-d963-499f-b96a-7215d6cbb8fa kind: Event lastTimestamp: "2026-07-01T21:07:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:05Z" name: error-404-isvc-7cf3f.18be467680546a99 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: e97d0b00-7087-4971-94a8-3beda97d5d3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 85747e84-d963-499f-b96a-7215d6cbb8fa kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-7cf3f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7cf3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f.18be4676813bbfa6 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: cf7d6685-06d9-45a9-9da3-4c3628b050a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 85747e84-d963-499f-b96a-7215d6cbb8fa kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7cf3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-7cf3f.18be4676813c0760 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 5bfe4e4d-9998-4e0a-8e0c-e89afe247ab5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 85747e84-d963-499f-b96a-7215d6cbb8fa kind: Event lastTimestamp: "2026-07-01T21:07:05Z" message: InferenceService [error-404-isvc-7cf3f] is Ready metadata: creationTimestamp: "2026-07-01T21:07:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:05Z" name: error-404-isvc-7cf3f.18be4681d3acb9d7 namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: de448040-cc80-4aa8-8830-aad9ddc57e25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:17.677087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-86dc6-predictor-6869d476d6-pldxc to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be468491909e54 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: dc97ebfe-d750-4e42-b7d8-9cd67ebfa28b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-86dc6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be46849c057153 namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: b3d92129-dd80-4e96-a6c4-3c7f74b95c4c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684cedb37b9 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: ecb86791-f6c0-4f5f-9516-ac355b88ded9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d045cb1a namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: c30b2fac-875b-4595-ac13-987c335106e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d34fcdea namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: d29be0d7-d15f-4177-a40d-514901dab1d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d3f5a269 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 7fd5420b-6035-4d75-a2a0-e28b66c1bc40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d3fe6eb2 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 02879237-7fca-4b06-b62f-eb48d5c3038b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d6d7eeb4 namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 93cd897b-9ecd-4e82-b383-23c850d95c9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684d7475f6d namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 84b224b1-771f-4e91-8cfe-1a8277e70942 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:16:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:35Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4684e4432d30 namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 74697cbf-2633-4ab4-a9fa-891fa48b6a0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4705d3d34ae1 namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: 17e5314a-3ee0-4e67-a2ef-286da60fe70f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be4705d3d3ace6 namespace: kserve-ci-e2e-test resourceVersion: "25398" uid: e87ff002-a468-4c56-bb26-06e941805d31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 5f281e07-2bbe-4193-9472-ac2de18a483d kind: Event lastTimestamp: "2026-07-01T21:16:35Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:35Z" name: error-404-isvc-86dc6-predictor-6869d476d6-pldxc.18be470658c7ad95 namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: 70699628-f46a-4a36-a6d1-2b761e9f14c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-86dc6-predictor-6869d476d6 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 1f84ff26-7b5e-4482-b2c0-805c15ed2119 kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'Created pod: error-404-isvc-86dc6-predictor-6869d476d6-pldxc' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6-predictor-6869d476d6.18be4684910f8e92 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: f56c3c8e-8c49-4d8d-bc3e-3fe7a90cd0f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: 14437c72-8ed2-4e12-aecc-647e9ae49d48 kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Scaled up replica set error-404-isvc-86dc6-predictor-6869d476d6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6-predictor.18be46848f976492 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: bdadf927-ef69-412a-ad1a-ecb564122248 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:07:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: ae1e295d-0ad6-4d5c-9614-f11da2fc2b2e kind: Event lastTimestamp: "2026-07-01T21:08:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:17Z" name: error-404-isvc-86dc6-predictor.18be46880e145c4f namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 381e483a-8eff-47f5-b927-fdda96bfac16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:07:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: ae1e295d-0ad6-4d5c-9614-f11da2fc2b2e kind: Event lastTimestamp: "2026-07-01T21:08:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:17Z" name: error-404-isvc-86dc6-predictor.18be46880e7261a8 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 8a4254b8-0c43-4ca5-b561-0cc522f33e44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: ae1e295d-0ad6-4d5c-9614-f11da2fc2b2e kind: Event lastTimestamp: "2026-07-01T21:08:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:47Z" name: error-404-isvc-86dc6-predictor.18be4699884f3f19 namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: a60be9e5-38e6-4f24-ad23-12493fcdc865 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: ae1e295d-0ad6-4d5c-9614-f11da2fc2b2e kind: Event lastTimestamp: "2026-07-01T21:08:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:47Z" name: error-404-isvc-86dc6-predictor.18be469988b84ba2 namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 0635b7d8-ea58-4189-ba01-f4af37556b71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 3ce2f030-a2e8-4021-96e9-bb0efa94ee8d kind: Event lastTimestamp: "2026-07-01T21:08:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:05Z" name: error-404-isvc-86dc6.18be46848f9fdae8 namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: e659cce9-b9ba-43de-8c88-1f3753c64319 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 3ce2f030-a2e8-4021-96e9-bb0efa94ee8d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-86dc6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-86dc6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6.18be46849012a9ad namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 3b4b1ed7-9f24-4de8-9344-62c43e9679f2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 3ce2f030-a2e8-4021-96e9-bb0efa94ee8d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-86dc6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-86dc6.18be46849012eab6 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 04b269ea-f98f-4793-88f9-e69003d8aeea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 3ce2f030-a2e8-4021-96e9-bb0efa94ee8d kind: Event lastTimestamp: "2026-07-01T21:08:05Z" message: InferenceService [error-404-isvc-86dc6] is Ready metadata: creationTimestamp: "2026-07-01T21:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:05Z" name: error-404-isvc-86dc6.18be468f9fb5687b namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 8aceff47-16eb-4ac1-a5b5-8f33cbc41f52 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:57:02.472425Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f554827459 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 3578b6dd-9cd0-460b-9738-d29da934914d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f56f209bc2 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 20b3a341-e537-4714-a2cf-c5db693e321d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f570795679 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: a9e13566-f4d6-42f4-a675-b79234814138 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f573380cc0 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 905a4719-e3b9-4990-bcb8-e6b2ecda6405 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:03Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f573a8972b namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: da2bdb72-0e33-4620-afe6-e6e41188890a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:03Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f573b2c2bd namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 9b906cc0-47ad-4ca4-8dd9-82edd616a031 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:03Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f577bf848a namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 181f81cf-5d7b-491a-b1b2-37dbb1740f9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T20:57:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:03Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f5785157e7 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 15fbb8ea-9faa-4e2d-a9a2-ea7b33db38f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T20:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T21:06:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:19Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be45f59616d84b namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: ac46c57f-644a-496b-a30e-b21c4a80db2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be46767948bf44 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 42864ef7-a180-4e49-9fb3-0b962b43395f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be467679496e00 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 4c80e3ec-01e8-43d3-92c0-5433c26abc8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 361e65bc-14bb-4657-a4ea-85dedccfc845 kind: Event lastTimestamp: "2026-07-01T21:06:19Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:19Z" name: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7.18be46770ac7e625 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 6bed56fa-c521-49a3-be50-bbbd49e3a64b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a93e1-predictor-85bdd6c986 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: d7e0af08-9ccf-4348-b6cc-b5f0a5d27399 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Created pod: error-404-isvc-a93e1-predictor-85bdd6c986-f5nv7' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor-85bdd6c986.18be45f553f340c5 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: b254ce55-cc8c-4989-ab9d-f1ecd54089e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 9c3ad73e-ffa2-41a5-b89c-a8c7d3f99b5f kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Scaled up replica set error-404-isvc-a93e1-predictor-85bdd6c986 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1-predictor.18be45f552526303 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: a56982ea-1c9b-46ce-932f-dcd381a00ca3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: c4dece85-368b-4ba6-b7fc-2c5ad7d11f3c kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: error-404-isvc-a93e1-predictor.18be45f8d0ab6789 namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 7d3a948f-b0a6-4505-908a-6f7d43d2514a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: c4dece85-368b-4ba6-b7fc-2c5ad7d11f3c kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: error-404-isvc-a93e1-predictor.18be45f8d127be99 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 76b455d1-6684-4f47-9953-03cf7b96689b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: c4dece85-368b-4ba6-b7fc-2c5ad7d11f3c kind: Event lastTimestamp: "2026-07-01T20:58:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:17Z" name: error-404-isvc-a93e1-predictor.18be45fc4fe4e881 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: fd03ac8c-27bb-4844-9a6e-bbe937e83208 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: c4dece85-368b-4ba6-b7fc-2c5ad7d11f3c kind: Event lastTimestamp: "2026-07-01T20:58:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:17Z" name: error-404-isvc-a93e1-predictor.18be45fc505776bd namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 06ce9b48-496c-4c99-b70a-ee2216f72dd9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 7a7c7f66-0480-4a5a-b342-8b03feb9b60e kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:49Z" name: error-404-isvc-a93e1.18be45f5524d7b37 namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: a9f1b70c-7ed4-4dee-bf06-1008c4fb291e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 7a7c7f66-0480-4a5a-b342-8b03feb9b60e kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-a93e1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a93e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1.18be45f552d241b7 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 13ade9f8-6941-479c-8c97-1a53e132e78e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 7a7c7f66-0480-4a5a-b342-8b03feb9b60e kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a93e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-a93e1.18be45f552d28906 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 7d9e4a17-6559-4776-9532-a975726a9287 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 7a7c7f66-0480-4a5a-b342-8b03feb9b60e kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: InferenceService [error-404-isvc-a93e1] is Ready metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:49Z" name: error-404-isvc-a93e1.18be4600510f0f6e namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 8c81be2a-49a9-4818-b3d8-d7347c6adfcd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:37.526352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c5814-predictor-5c7686f9f4-mclpb to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be468930ac4dc6 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 5aeb67f5-9b62-4ae8-b56f-dd47687ed7d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46894a21a501 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 40810826-b55e-455d-a873-563981c6b74b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46894b8a23e1 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 9518edf6-5dd3-4a36-af67-c042e2b916a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46894e6adf2e namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: fdc8fe8a-ce0f-4cc9-b80b-1c3bc749245d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46894eda9154 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 6608aea4-c83f-45ce-9200-4076aef1d06f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46894ee4d36f namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 727652be-c4bf-42dd-88b7-6c034c673484 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be46895166843c namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 2c96dd7a-6a5c-4dd3-b148-19d5cf5be6ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be468951d34627 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 31464239-96c0-4528-82c7-08d77d9e10a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:16:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:55Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be4689927fe1e0 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: 5254ae2d-d129-44be-83e0-e88bc0d116f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be470a64e36ce8 namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: 28fd3b3a-c9a7-44b0-87dc-748f82852f9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be470a64e41ba4 namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: b876d9c9-bb85-4f9c-8c54-c1d44dda0e24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76aa05e4-cee0-4c7b-bbde-5a1c3bfea65b kind: Event lastTimestamp: "2026-07-01T21:16:55Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:55Z" name: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb.18be470b06de6b88 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: e8e55b36-1159-42a4-898c-a7b7d288af3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c5814-predictor-5c7686f9f4 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 52d86095-0ec1-442a-9f52-d1245fdcab11 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'Created pod: error-404-isvc-c5814-predictor-5c7686f9f4-mclpb' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814-predictor-5c7686f9f4.18be4689303e4530 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: e983261a-31bd-40ca-89b1-95233dd9b198 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 9a5d38a1-3619-4fe7-96d8-3c36becf2e69 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Scaled up replica set error-404-isvc-c5814-predictor-5c7686f9f4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814-predictor.18be46892eb9e752 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: e166c7ac-37a7-4941-8c93-f30095ef1cc7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: f8382f0a-6f5a-4945-961c-7545fba7a64c kind: Event lastTimestamp: "2026-07-01T21:08:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:07Z" name: error-404-isvc-c5814-predictor.18be468cad28960f namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 4c9ab511-5264-4fcb-9506-156c9b0ee19e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: f8382f0a-6f5a-4945-961c-7545fba7a64c kind: Event lastTimestamp: "2026-07-01T21:08:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:07Z" name: error-404-isvc-c5814-predictor.18be468cad907acf namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: f54722da-8025-4281-bdd4-0ecf98677b09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: f8382f0a-6f5a-4945-961c-7545fba7a64c kind: Event lastTimestamp: "2026-07-01T21:08:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:52Z" name: error-404-isvc-c5814-predictor.18be4693aafda1e3 namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: f31a61b9-b051-4a51-95d6-d9cb66f9d1bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: f8382f0a-6f5a-4945-961c-7545fba7a64c kind: Event lastTimestamp: "2026-07-01T21:08:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:52Z" name: error-404-isvc-c5814-predictor.18be4693ab49e505 namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: 539e2903-4b8a-4cab-9a00-923fe2b0d1c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 8fb215de-7400-4783-b3e6-8ba78beab0d9 kind: Event lastTimestamp: "2026-07-01T21:08:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:25Z" name: error-404-isvc-c5814.18be46892ed01b7b namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 1037e83e-d7c0-40c0-8583-815f72c0b2c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 8fb215de-7400-4783-b3e6-8ba78beab0d9 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-c5814": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c5814": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814.18be46892f30f70c namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: b8b1f2da-23e5-41f1-a56b-f9aa38422e94 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 8fb215de-7400-4783-b3e6-8ba78beab0d9 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c5814": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:37Z" name: error-404-isvc-c5814.18be46892f312d4e namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 6b7280fa-38c3-4c82-9a56-b2e9364cc979 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: 8fb215de-7400-4783-b3e6-8ba78beab0d9 kind: Event lastTimestamp: "2026-07-01T21:08:25Z" message: InferenceService [error-404-isvc-c5814] is Ready metadata: creationTimestamp: "2026-07-01T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:25Z" name: error-404-isvc-c5814.18be46944e172efd namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: f4c4363e-e6db-47a3-bf9a-5df589cd8d02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:16:52.385146Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a60d5ab40 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 86433dcb-ea8b-4df3-a62c-a7e8b4ca1b2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a7bf255a0 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: df776be8-4701-4996-bf8a-ee02211cb4d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a7d60b1c6 namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: cbbb29fc-30bb-4866-8a7e-bb27472796ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a80556598 namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: ac0474ea-9432-44c5-9908-b6669aff8d7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a80c8805e namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 9dff80c1-164a-462b-a88e-61989ae3f554 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a80d2ad69 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: e894d837-3b52-4a6c-b4e3-482123aae05d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a83ab4183 namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: 87bedf8e-456f-4015-bd28-6ef89321530b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470a841d7cae namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 38b58f95-2cea-4b04-9f90-5093df909005 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:17:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:30Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be470ad6bf572e namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 02c2986b-e907-4a9a-8f01-6bdfaab856a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:34:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:21Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be47feb180a2d2 namespace: kserve-ci-e2e-test resourceVersion: "33810" uid: f1513c29-f1ad-44a7-851f-cd0519707c9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d60c38a1-a61b-47d6-8da7-8584afb9c4b9 kind: Event lastTimestamp: "2026-07-01T21:34:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:21Z" name: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m.18be47feb1811ce1 namespace: kserve-ci-e2e-test resourceVersion: "33811" uid: 3a42301e-311e-4134-9de7-c1a0b98d727b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ce993-predictor-59dfbf46cd namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: e21a7b20-4681-4c1c-a4c6-801f87b916b8 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created pod: error-404-isvc-ce993-predictor-59dfbf46cd-d8h5m' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor-59dfbf46cd.18be470a607daf6a namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 2524bba7-815f-4534-b3e2-30c6dd16414e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 26d48d3b-5fb4-4591-8d58-bf8a3b3e2aae kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Scaled up replica set error-404-isvc-ce993-predictor-59dfbf46cd from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993-predictor.18be470a5ece99ee namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 0b3e1d16-e8e8-473c-af38-9eca4cbcf8eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 1fd7749a-a23c-4049-8034-9d193921a3c7 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: error-404-isvc-ce993-predictor.18be470ddd729943 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 682f8735-686e-454d-ac23-6b86b442cb62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 1fd7749a-a23c-4049-8034-9d193921a3c7 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: error-404-isvc-ce993-predictor.18be470ddddb11de namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 180e7ef5-abb5-4754-aa64-db96d0ea9ace reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 1fd7749a-a23c-4049-8034-9d193921a3c7 kind: Event lastTimestamp: "2026-07-01T21:17:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:22Z" name: error-404-isvc-ce993-predictor.18be47115ccd5c45 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: f6d69da7-1a7d-4f36-a3b1-d1026fc852e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 1fd7749a-a23c-4049-8034-9d193921a3c7 kind: Event lastTimestamp: "2026-07-01T21:17:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:22Z" name: error-404-isvc-ce993-predictor.18be47115d53686d namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 8e692b2f-8346-47ef-9ad6-b26ae459803b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: a381ce3b-ed48-4974-ae03-60bf3e7813bf kind: Event lastTimestamp: "2026-07-01T21:17:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:40Z" name: error-404-isvc-ce993.18be470a5ed54343 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 830aec39-4fe2-4dda-8cba-62fc86e6b60f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: a381ce3b-ed48-4974-ae03-60bf3e7813bf kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-ce993": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce993": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993.18be470a5f590b50 namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 103404e0-6153-4067-a44c-24e819b7b877 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: a381ce3b-ed48-4974-ae03-60bf3e7813bf kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce993": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:16:52Z" name: error-404-isvc-ce993.18be470a5f59660f namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: fede4d3f-f724-4e57-83bf-66e49a87f14a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: a381ce3b-ed48-4974-ae03-60bf3e7813bf kind: Event lastTimestamp: "2026-07-01T21:17:40Z" message: InferenceService [error-404-isvc-ce993] is Ready metadata: creationTimestamp: "2026-07-01T21:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:40Z" name: error-404-isvc-ce993.18be4715932afdac namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: 1e4adc51-99d7-4f8b-bc40-355f0421240d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:05:57.419032Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f1d61-predictor-7896dd4c76-6snkv to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be4671e1cfe374 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 074659e2-4578-48fc-9e1d-2e0e5125387a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f1d61-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be4671eaf3a171 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 1d2a7c74-aa00-4a36-a393-66dcfd162876 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be46721efdb6ee namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 009bc43b-076e-4023-b057-ff48053792fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be4672205c42db namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: 818de697-fc20-4581-973d-c10eecd36c27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be467223a9a415 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 7e4d2116-c475-47f8-9b56-4c2688294a17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be4672241b03f0 namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 13c78d49-afa3-4f18-b26f-26329b802995 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be467224249da5 namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 8683b257-2e34-41a4-813e-efaf2b15ff32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be46722873b791 namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: 0834f80f-e90a-4f52-9a86-5bfa618ae483 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:05:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:58Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be467228e555e9 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 154e66e3-1ab9-4a68-80df-544205c3dff6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:06:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:44Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be46722eac898e namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: e2e1e6f1-7ed5-4ad5-930c-9f08350b0ef3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be468494d667b7 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: c9a5d04d-3277-496c-88d1-3d94f6cdf2c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be468494d6f2b7 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 527a0410-8c53-4d67-a30b-a1d78efe3ba3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: fad15195-1882-4c1a-8731-cc759e2831db kind: Event lastTimestamp: "2026-07-01T21:07:19Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:19Z" name: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv.18be46850ae7caba namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 0867ee4e-2a88-42c2-8b82-347cd875b8dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f1d61-predictor-7896dd4c76 namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 95948c5b-d0ba-47ea-9a98-f68274e1741e kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Created pod: error-404-isvc-f1d61-predictor-7896dd4c76-6snkv' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61-predictor-7896dd4c76.18be4671e15d01f0 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 0afc651c-8ae6-4b0a-9493-d34bba1c9820 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: ea2b76e3-21d2-4d1d-82c5-02201a442bd9 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Scaled up replica set error-404-isvc-f1d61-predictor-7896dd4c76 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61-predictor.18be4671dfc1b350 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 2722b1f2-ee17-424e-9d74-e1cb8b3cea1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 7eda50cb-ffaa-44ec-9633-69bfbef2000c kind: Event lastTimestamp: "2026-07-01T21:06:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:27Z" name: error-404-isvc-f1d61-predictor.18be46755e4e52c1 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 0be298e2-8737-4a34-9550-43415a9c52a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 7eda50cb-ffaa-44ec-9633-69bfbef2000c kind: Event lastTimestamp: "2026-07-01T21:06:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:27Z" name: error-404-isvc-f1d61-predictor.18be46755e9ef2d5 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 2065db04-c19a-4cb0-9dd4-e4fd126a70fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 7eda50cb-ffaa-44ec-9633-69bfbef2000c kind: Event lastTimestamp: "2026-07-01T21:07:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:12Z" name: error-404-isvc-f1d61-predictor.18be467c5c0fe857 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: e3792888-d17d-4100-9f0c-ff5247bc6787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 7eda50cb-ffaa-44ec-9633-69bfbef2000c kind: Event lastTimestamp: "2026-07-01T21:07:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:12Z" name: error-404-isvc-f1d61-predictor.18be467c5c774063 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: ebddf7e6-6d76-4801-a86c-08908aedc40b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 4a217f33-7b90-450d-ae61-50da5151c495 kind: Event lastTimestamp: "2026-07-01T21:06:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:54Z" name: error-404-isvc-f1d61.18be4671dff8f211 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 14ad7186-bfed-45d9-97db-326e3c2adbe6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 4a217f33-7b90-450d-ae61-50da5151c495 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-f1d61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61.18be4671e0642da2 namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 92c64c40-7bfa-47e1-82be-61c22e27e60d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 4a217f33-7b90-450d-ae61-50da5151c495 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f1d61.18be4671e0646e04 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: e6c72230-4473-42a2-9d3a-b74d6e60c46c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 4a217f33-7b90-450d-ae61-50da5151c495 kind: Event lastTimestamp: "2026-07-01T21:06:54Z" message: InferenceService [error-404-isvc-f1d61] is Ready metadata: creationTimestamp: "2026-07-01T21:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:54Z" name: error-404-isvc-f1d61.18be467f3de62bd2 namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 3cb7457e-f208-4bff-84e1-989bfa26379b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:56:32.438187Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee56545416 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 49f0bcf4-cf68-4c21-b968-0c9defa27358 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee70824b82 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: cb836b46-eefc-4c85-94d3-319a264e86f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee72a69ff8 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 22b70897-bd3e-400d-baca-b885e2c1a435 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee75940d8b namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: a62dcbff-be03-4ce4-8927-42b97240e2d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee75f696ec namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 2ae4e3da-3b95-4201-bdd6-bf5cfad2b02f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee75ff8b2e namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: a88d2dc4-0bb6-4f56-a39b-f634a35d0f3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:33Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee78f929f6 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 25b8a7a4-5354-423a-9538-3b8c5a5c0fe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T20:56:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:33Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee7954b7d7 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: c4f4918c-33bb-4103-a7ac-5e975f24acdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T20:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T21:05:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:59Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be45ee9183e492 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: eee67296-633b-4c77-b98e-521863070ed6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be4671e45de698 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: d6438dd8-a767-4ce0-9461-ace356f10b5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be4671e45e4d60 namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: c5ff5781-bdb0-4d8b-b379-89598ca575aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72b059bc-a318-419f-b477-f0d1b982dcac kind: Event lastTimestamp: "2026-07-01T21:05:59Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:59Z" name: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7.18be467259f112f4 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 864ff01f-332e-469a-a72f-768418187b28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f347e-predictor-5965cb5bdb namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 450b4562-bbc6-4c3d-8ab0-a9c56fdded75 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Created pod: error-404-isvc-f347e-predictor-5965cb5bdb-ztwl7' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor-5965cb5bdb.18be45ee55be2b7e namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 4bfb3540-b0ff-4c16-ae84-53254a148c1f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: 6566e7a9-b539-4deb-a0af-88fd91432a0e kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Scaled up replica set error-404-isvc-f347e-predictor-5965cb5bdb from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e-predictor.18be45ee5439f10f namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 3a04bbf1-7add-470c-a880-acbde901f5e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: c3cea3f0-488c-4787-bdeb-2ca04ed03978 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-f347e-predictor.18be45f1d290dfda namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 66edd636-50f7-47e1-b1f3-358b855796ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: c3cea3f0-488c-4787-bdeb-2ca04ed03978 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: error-404-isvc-f347e-predictor.18be45f1d2f6f3f1 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: da1860f5-a6db-4377-bf21-58eaa4479af0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: c3cea3f0-488c-4787-bdeb-2ca04ed03978 kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: error-404-isvc-f347e-predictor.18be45f8d0a3136a namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 1f326478-6238-47a5-a446-eb82675665c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: c3cea3f0-488c-4787-bdeb-2ca04ed03978 kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: error-404-isvc-f347e-predictor.18be45f8d1256235 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 888a1ea5-a286-4b38-ba25-798de4333b31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: a307b00b-156c-4da4-a098-2bcf7e19ff44 kind: Event lastTimestamp: "2026-07-01T20:57:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:19Z" name: error-404-isvc-f347e.18be45ee54435241 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: dd18c1f1-9572-48c2-80c2-33fb5afe1788 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: a307b00b-156c-4da4-a098-2bcf7e19ff44 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-f347e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f347e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e.18be45ee54c7f46b namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 188f873b-9dd8-45d0-84b1-4cc85a04fa81 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: a307b00b-156c-4da4-a098-2bcf7e19ff44 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f347e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:32Z" name: error-404-isvc-f347e.18be45ee54c8d522 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: c330bc18-19fb-4e43-bf6c-9bd585889a2b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: a307b00b-156c-4da4-a098-2bcf7e19ff44 kind: Event lastTimestamp: "2026-07-01T20:57:19Z" message: InferenceService [error-404-isvc-f347e] is Ready metadata: creationTimestamp: "2026-07-01T20:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:19Z" name: error-404-isvc-f347e.18be45f94c4bb75c namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 883e42b5-e75d-4a9d-8c9c-2334ea52e7ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:54:32.138003Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d253e1488e namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: e487f46e-fef2-4b5b-b061-180ef8d4f4bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d2653f1a63 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 0d42be53-fecf-41a1-893b-1c0ee18b0701 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d291914a6d namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 08587600-3a86-48de-82be-74b735a47422 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d29312655e namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: 07cfce10-d294-4d4f-97ec-e857c898481b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d29583a005 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 024a02f0-6855-47cf-9306-6a7317d58031 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d295e6ffab namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 05270672-13dc-4f5b-8d29-7eabf4964fca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:54:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" metadata: creationTimestamp: "2026-07-01T20:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:37Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45d3a793f387 namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 2cc71bcf-561f-4687-95f1-d08c14f70779 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" in 30.917s (30.917s including waiting). Image size: 1555648931 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:08Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45dada6881cc namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 9ece8a62-d6db-4744-9d71-5d39e8197ae7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45daef031c4b namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 2cf1fd71-2517-4f59-85a8-f26cbcee12e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45daf02c87ee namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: e215390a-ec51-42c5-9739-c331e9b9269f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45daf03994bd namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: e7808410-0aa7-44f1-925f-d6ed55dd452d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.375s (4.375s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45dbf5062af0 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 92b8575f-0910-49f8-86fa-d906118ad3f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45dc0034eb14 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 7035367f-aad0-408b-ac10-e0ed047f9e48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45dc00af0f68 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: 4f2b4b13-beec-4b2d-8adf-eef3f45a9fe0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:56:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:21Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45dc54497654 namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: ae4f0a94-d5c2-48d6-9ff3-d3a4864c0d2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45f54e667d10 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 70779193-d703-4766-a420-d85ae83dd8fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45f54e66f13e namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 0e88e480-4f4d-49e1-bf43-448b2ec4f648 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 603c3269-3a3d-4ff6-b52a-41bb9af0d5f4 kind: Event lastTimestamp: "2026-07-01T20:57:06Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:06Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2.18be45f62cb66f5b namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: c636788c-f1cf-4dde-a604-14a8b8aefdef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6f5c857d79 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 680a46b3-e107-426b-a911-ea8aa2b7adfe kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6f5c857d79-5p5g2' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79.18be45d25384dc3f namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ac2ae5dc-719c-4459-9d95-497fd4da4cb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: f50d9845-75e9-49bc-9eb1-f6343eab6e30 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6f5c857d79 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-1-predictor.18be45d24f40c148 namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 513ec220-d778-4975-b78d-f4438a643bae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 0e5c56ec-e706-4843-a9f3-3affdc2124cc kind: Event lastTimestamp: "2026-07-01T20:55:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:17Z" name: isvc-sklearn-graph-1-predictor.18be45d5d4ca749a namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 3e495bc0-0435-4dcc-95b5-0fefa9e20327 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 0e5c56ec-e706-4843-a9f3-3affdc2124cc kind: Event lastTimestamp: "2026-07-01T20:55:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:17Z" name: isvc-sklearn-graph-1-predictor.18be45d5d532bb7b namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 483a9799-7b49-4169-a4e1-8854347b2150 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 0e5c56ec-e706-4843-a9f3-3affdc2124cc kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-sklearn-graph-1-predictor.18be45e052737d28 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: e3822130-38b0-42f8-a808-641fc0af922a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 0e5c56ec-e706-4843-a9f3-3affdc2124cc kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-sklearn-graph-1-predictor.18be45e052db06f7 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: c403076c-6159-4354-a0e6-142b84ba8396 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 0e5c56ec-e706-4843-a9f3-3affdc2124cc kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: deployments.apps "isvc-sklearn-graph-1-predictor" not found metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: isvc-sklearn-graph-1-predictor.18be45f54be133ef namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 5c67f741-ef11-4f0b-af90-1ccefd0a0a95 reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: cec99524-4cd0-43bf-b124-77a85b9169ed kind: Event lastTimestamp: "2026-07-01T20:56:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:31Z" name: isvc-sklearn-graph-1.18be45d25567f0ec namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: f2574c69-ea1f-4ed4-9b1b-96cb06729492 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: cec99524-4cd0-43bf-b124-77a85b9169ed kind: Event lastTimestamp: "2026-07-01T20:56:31Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T20:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:31Z" name: isvc-sklearn-graph-1.18be45ee0c5a6ee8 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 9f1ddbab-a3a6-4d48-ae30-5d938c846596 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:54:32.624267Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d270dcfac5 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: cb4a40b5-6410-49ee-a9f5-93c11f0d19d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d28b32660e namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: 2cadc86a-0ba3-417d-9a66-8b1c41ec7157 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d28cd501ee namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 7b6d3034-ded9-4d07-98fd-39cd25964c32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d29073d668 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 4e707143-e124-410e-811c-c7ebc4ea5cdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d290f7d1b3 namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: cf50e534-3b26-4a04-b3af-4e15d25c56eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:54:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" metadata: creationTimestamp: "2026-07-01T20:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:37Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45d3a7cc6cdf namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 93a8415b-0cfa-4b07-a743-2fac055cd3f5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" in 31.149s (31.149s including waiting). Image size: 1555648931 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45dae876edb1 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 801dcc41-1924-4306-b882-a783b53f93e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45daf06c5a76 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 8b7de986-7723-4430-9e71-6458ebdc7055 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45daf21fa4f6 namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: b630cd4f-8019-4017-ac8b-51cc5e9f02be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45daf22eb5fb namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: af8e30e1-feb6-49ff-8f85-a185324e7d59 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.61s (4.61s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45dc0501aae8 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 68f7af2f-ec97-4b7e-ba13-0d4200c23408 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45dc09456262 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: a1d3aa5c-18b3-4f13-af15-4e5926200c8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45dc09bbc7de namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 45228fad-8935-45e6-b18c-86dc0cb5629b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:56:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:30Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45dc18479e65 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ab6dbfe-e4e4-4cd0-aea1-ac283bff644f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45f54a149c6c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 118005af-269d-4383-9e1f-590cbd702405 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45f54a1519e4 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 71571e94-3702-4493-954a-2f84af9aa3f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e28fcffa-cde8-44ce-9015-a1ad49645e7a kind: Event lastTimestamp: "2026-07-01T20:57:05Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:05Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm.18be45f5f0c2af97 namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 57fef8da-4b56-4492-a0e5-0c85d558307f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6fb9776d89 namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 2afe6b6e-cd1b-4d99-8dc1-390713df6127 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6fb9776d89-npwbm' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89.18be45d2706507d8 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: 069305b3-def5-4d30-b185-4491905d98b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: 1eebbc11-c54f-43d6-80a2-2007a580777e kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6fb9776d89 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-sklearn-graph-2-predictor.18be45d26d4f797e namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 3d8606c9-b12f-45b3-9cf9-869138da855d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 9fb2c0a7-aaf6-4fd8-929f-e200933f25d7 kind: Event lastTimestamp: "2026-07-01T20:55:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:47Z" name: isvc-sklearn-graph-2-predictor.18be45d5ebd3f73a namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: cdebf339-214f-4636-8daa-6b6a4162d611 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 9fb2c0a7-aaf6-4fd8-929f-e200933f25d7 kind: Event lastTimestamp: "2026-07-01T20:55:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:47Z" name: isvc-sklearn-graph-2-predictor.18be45d5ec39265d namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: c95b4d50-2855-4b1b-bb69-cb167c4f8ffd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:56:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 9fb2c0a7-aaf6-4fd8-929f-e200933f25d7 kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-sklearn-graph-2-predictor.18be45e765a49c47 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 97669b5c-4b33-4968-a277-8498f88cf8ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:56:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 9fb2c0a7-aaf6-4fd8-929f-e200933f25d7 kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-sklearn-graph-2-predictor.18be45e7660c7a9f namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 4f7acfda-5775-4bfe-9e30-3d66ace48037 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 0de33886-979c-44be-b152-d3fd69704891 kind: Event lastTimestamp: "2026-07-01T20:56:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:40Z" name: isvc-sklearn-graph-2.18be45d26d561253 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: ed7b66a1-c21e-4c22-b82c-c46a0eab0772 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 0de33886-979c-44be-b152-d3fd69704891 kind: Event lastTimestamp: "2026-07-01T20:56:40Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T20:56:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:40Z" name: isvc-sklearn-graph-2.18be45f02467ed83 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: c196cb1d-f1da-4b5c-9534-01e8d8c22231 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:54:32.523546Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-9b7pg to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:54:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d26adc1f73 namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: bf4a90e5-57ff-4fc6-87c3-ec5ddddf734f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d277601b38 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 88e34d53-ad36-4159-bfce-37820c08abd5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:54:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d2aea4ac9c namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: ee778df1-1fe2-4ccc-bf12-1a9b5f060ad7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d2b1df41df namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: 7bf5c66e-c918-4fbf-bd02-e1215c7c73f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d2b94865d0 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 3221dd19-3b10-4895-b8a1-6223934c5d8a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d2bb443bc1 namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 75406c3f-e561-4e56-a488-43c7ccc8f960 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:54:39Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T20:54:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45d420203168 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: a171567c-e117-4002-b109-849529dc5a3f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.558s (29.558s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45db01f9d06b namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: 56531f06-898a-4dee-88d3-e3122120808d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45db06c7dbea namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 655fc03a-2bc3-454f-889b-5391d6f7cb36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45db079d2078 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: eb7731e4-1f2e-4f42-ada0-8f010a31cfe8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45db07a9fed2 namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: ee9741e9-9075-4c88-83d8-208602fcd7d5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.114s (4.114s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45dbfce289c7 namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 99bc36c4-d713-497a-8fb4-0e7eb89dbef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45dc00e5d38a namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: d11c135f-7654-4592-9758-d6d7fe427203 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45dc01cca055 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 3852c43a-9c09-4c6a-9747-e07717048561 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T20:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:56:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45dc5447d263 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: ac0d6a89-905f-4e98-b527-fa21216f1ee8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45f558a5f2bf namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 924ab6b1-fa23-47e1-8e9d-0ac902730b4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45f558a640c5 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 5ce3b855-e63d-4c17-83d1-5219e389acab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 140bfc0f-a451-4a3b-9acc-1fdcde1702e5 kind: Event lastTimestamp: "2026-07-01T20:57:06Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-9b7pg.18be45f62cbe17b2 namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 31f0785d-7b4c-4123-bf59-b69486e2a39e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 7f018381-57c1-4036-b3a5-9660f580db7a kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-9b7pg' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be45d26a80fca3 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 27d91b48-3319-4392-91fc-8d35a15a72fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 8e504d6d-fcb7-4e54-aa9b-44cc6411bf9c kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: isvc-xgboost-graph-predictor.18be45d267b078ab namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 37d04302-e15c-4c7c-a5c5-601402afda06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: aa4aa7c3-60ee-4209-ab95-4c728fb6766c kind: Event lastTimestamp: "2026-07-01T20:56:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:02Z" name: isvc-xgboost-graph-predictor.18be45d5e642a27c namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 0b35e2f2-c8cb-4d42-b57a-8b473b9ff292 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: aa4aa7c3-60ee-4209-ab95-4c728fb6766c kind: Event lastTimestamp: "2026-07-01T20:56:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:02Z" name: isvc-xgboost-graph-predictor.18be45d5e6ac0e92 namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 9e569f4c-b09e-4b73-9ce8-46c625ba0b90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: aa4aa7c3-60ee-4209-ab95-4c728fb6766c kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-xgboost-graph-predictor.18be45eadf273732 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 9ee1fab3-699c-4949-a79c-ea7554fe52a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: aa4aa7c3-60ee-4209-ab95-4c728fb6766c kind: Event lastTimestamp: "2026-07-01T20:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:47Z" name: isvc-xgboost-graph-predictor.18be45eadfafc5df namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 15395f88-f6e0-4d79-9234-c6126446ad54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: d0e16b95-1383-4e55-b082-32143d62086a kind: Event lastTimestamp: "2026-07-01T20:56:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:21Z" name: isvc-xgboost-graph.18be45d267bb369e namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: aa6ee705-87b5-4b80-ad98-f28d9940bb5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: d0e16b95-1383-4e55-b082-32143d62086a kind: Event lastTimestamp: "2026-07-01T20:56:21Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T20:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:21Z" name: isvc-xgboost-graph.18be45ebb7fed5d8 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: ff2b2158-3cbd-4692-bfde-b933c7be5e5c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:56:52.064398Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-794cbf94f-6krlx to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T20:56:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:56:52Z" name: model-chainer-794cbf94f-6krlx.18be45f2e8247b3b namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 609ff0d2-354c-42d5-a781-3326c0d40420 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:56:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:52Z" name: model-chainer-794cbf94f-6krlx.18be45f2f36a0959 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 3e89ecdc-f79b-4ff8-b07f-dd92e6dfe37c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:53Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:56:53Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:56:53Z" name: model-chainer-794cbf94f-6krlx.18be45f32564a173 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: aa651271-f078-41f6-b5ee-505304fea11a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:56:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:53Z" name: model-chainer-794cbf94f-6krlx.18be45f326b0a593 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 582cbf37-1abd-4b08-92a1-8d2e10f149d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:56:53Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:53Z" name: model-chainer-794cbf94f-6krlx.18be45f329a44c89 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: e8754dce-5c03-443d-a5fe-c86a44fc04c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:56:53Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:53Z" name: model-chainer-794cbf94f-6krlx.18be45f32a1ab49b namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: 99685d2a-0ae8-48d1-a3e5-7c87a40bae98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: model-chainer-794cbf94f-6krlx.18be45f54390fd56 namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: f6e8950d-fc63-4743-b846-072f442b23f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-794cbf94f-6krlx namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 64ef75e8-c42f-40bb-9494-61c574ce3025 kind: Event lastTimestamp: "2026-07-01T20:57:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:29Z" name: model-chainer-794cbf94f-6krlx.18be45f5c3394009 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 8f6d56c6-db18-432f-9bac-2d50ca0e9605 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-794cbf94f namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 86151c2f-528f-4e28-8859-1bf40fbd0270 kind: Event lastTimestamp: "2026-07-01T20:56:52Z" message: 'Created pod: model-chainer-794cbf94f-6krlx' metadata: creationTimestamp: "2026-07-01T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:52Z" name: model-chainer-794cbf94f.18be45f2e799a4f2 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: dc71c77a-9fcb-4609-b9a1-f423c8f684fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: efc3a875-5b23-48a6-bc49-bf814b8dc4f9 kind: Event lastTimestamp: "2026-07-01T20:56:52Z" message: Scaled up replica set model-chainer-794cbf94f from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:52Z" name: model-chainer.18be45f2e6c78ce1 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 859e19a5-bc2f-45d4-9d3d-6c4dfd014203 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 68855a9b-a4d1-4da4-9aca-bff67ee66031 kind: Event lastTimestamp: "2026-07-01T20:56:52Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:52Z" name: model-chainer.18be45f2ec089066 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 53583637-d2b4-43e7-9a6d-5e2769fa8dfb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:27.215619Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7cf3f-6ddb7696d5-hzlbm to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:27Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be4686ca1afe10 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 5bad9194-f4fc-4783-9ee2-a68dd1f336d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7cf3f-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:27Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be4686d3f17718 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: b258a6c2-beec-40d1-b51e-5e2e1f258e14 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:28Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:28Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:28Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be468707649108 namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: d8f175a1-1859-412d-b6d7-9b30b0c41c19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7cf3f} kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:28Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be46870887cbde namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: 2ce45939-e6ad-4c16-8ce5-b8ab9c8e6c61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7cf3f} kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:28Z" message: 'Created container: sequence-graph-7cf3f' metadata: creationTimestamp: "2026-07-01T21:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:28Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be46870b4ce371 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 6a465182-e9e6-49bb-9a6c-9711a991cc34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7cf3f} kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:28Z" message: Started container sequence-graph-7cf3f metadata: creationTimestamp: "2026-07-01T21:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:28Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be46870bbe2982 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 85f9223a-c361-4987-8ebf-dc6a4cef85cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7cf3f} kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Stopping container sequence-graph-7cf3f metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be46892333421d namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 9479f9c3-2983-47e5-9cb9-50f75ea01ff5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7cf3f} kind: Pod name: sequence-graph-7cf3f-6ddb7696d5-hzlbm namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 45078883-6fc1-47bf-8657-f28d04d493f6 kind: Event lastTimestamp: "2026-07-01T21:08:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:04Z" name: sequence-graph-7cf3f-6ddb7696d5-hzlbm.18be4689c0f07693 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 4f512bf8-6923-4cd5-af73-30f3c461973e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7cf3f-6ddb7696d5 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 5b95fb48-6b78-4d93-b280-0939fb47d889 kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: 'Created pod: sequence-graph-7cf3f-6ddb7696d5-hzlbm' metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:27Z" name: sequence-graph-7cf3f-6ddb7696d5.18be4686c9a3d00d namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 8047ee60-1ecb-42b7-9706-4b61ff5e0bbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 2619af4d-adb3-49dc-90db-fb82a22c715c kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: Scaled up replica set sequence-graph-7cf3f-6ddb7696d5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:27Z" name: sequence-graph-7cf3f.18be4686c8f6eeac namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: f806770e-b50d-4dc7-ba52-a05268010dcd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 09d2d732-fae7-4241-ac21-b17ff8c19425 kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: InferenceGraph [sequence-graph-7cf3f] is Ready metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:27Z" name: sequence-graph-7cf3f.18be4686ce728ed1 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 9b2b8995-40a0-4642-a46d-d22e0e4915e9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:58:02.371261Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a93e1-84bdb6d745-qlk2l to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be460346c2250f namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: a297e362-5ee2-4f45-a79a-1eaf33033015 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be4603606bcaa4 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: f3640b8c-cc8d-485e-9823-72c538812b1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a93e1} kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be460361b4ca5d namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: a853b520-4aa5-4126-966f-8a866a5dd385 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a93e1} kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: 'Created container: sequence-graph-a93e1' metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be46036466fd39 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 34be3363-ed9d-4022-9d4a-a959b335caca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a93e1} kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: Started container sequence-graph-a93e1 metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be460364d2f52f namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 70c8119b-220f-4301-b0bf-28d84005f068 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a93e1} kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Stopping container sequence-graph-a93e1 metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be467672af5c8d namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 2fedd6fd-3285-44be-baac-014de603e980 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:06:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a93e1} kind: Pod name: sequence-graph-a93e1-84bdb6d745-qlk2l namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0b90842f-4019-43ef-bc18-c15222f07ecc kind: Event lastTimestamp: "2026-07-01T21:06:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:44Z" name: sequence-graph-a93e1-84bdb6d745-qlk2l.18be46770d221152 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 62275059-f7d7-463c-aab0-ddcf0b1b4e82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a93e1-84bdb6d745 namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: 5cea1781-a68f-4228-a80f-1266c3b97295 kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: 'Created pod: sequence-graph-a93e1-84bdb6d745-qlk2l' metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1-84bdb6d745.18be46034653595a namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: d5875c10-85ba-4fca-987d-ac26676fbc8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 1e14274e-2eed-416a-bc95-ed664abd10c6 kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: Scaled up replica set sequence-graph-a93e1-84bdb6d745 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1.18be460345a33d8c namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 5452c4c2-3dfe-4f07-98a7-c52fe1746753 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: fca40768-a3a1-489e-b979-d6dc598551d1 kind: Event lastTimestamp: "2026-07-01T20:58:02Z" message: InferenceGraph [sequence-graph-a93e1] is Ready metadata: creationTimestamp: "2026-07-01T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:58:02Z" name: sequence-graph-a93e1.18be46034b3ceac8 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: 4905860b-b4be-4cfd-aa3b-6cfcb844f10a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:58:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: a709252b-4cc1-458b-80da-54a009279962 kind: Event lastTimestamp: "2026-07-01T20:58:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:32Z" name: sequence-graph-a93e1.18be4606c42edb7e namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 80ef9a64-8302-4827-9731-ea9115ede546 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:58:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: a709252b-4cc1-458b-80da-54a009279962 kind: Event lastTimestamp: "2026-07-01T20:58:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:32Z" name: sequence-graph-a93e1.18be4606c4785856 namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 6a3c8d9f-c285-46f2-ba2b-5c3cc2d28b55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:08:37.534218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22012" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c5814-8549fd89d-pwg26 to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be4697296bb7ec namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: 5a8ec1be-5ce8-43f7-b50f-62bbd907edcd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be469743731d21 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 4439fead-51e0-4d5c-afbc-8362cc092ab1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5814} kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be469744ba9586 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 07de414b-7c9b-4160-a572-f23f4a3cecdf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5814} kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:08:38Z" message: 'Created container: sequence-graph-c5814' metadata: creationTimestamp: "2026-07-01T21:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:38Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be469748bb0e25 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: ba58c032-dbd3-430a-8c3b-81424091ab7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5814} kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:08:38Z" message: Started container sequence-graph-c5814 metadata: creationTimestamp: "2026-07-01T21:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:08:38Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be469749347958 namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: 6bea4456-a807-4f84-b3d1-51fdb32b04ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5814} kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Stopping container sequence-graph-c5814 metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be470a52b5ca43 namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: b02b08fc-29a1-4b7a-a11e-dde1a1390030 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5814} kind: Pod name: sequence-graph-c5814-8549fd89d-pwg26 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 5bfe64ae-33f2-40a0-b9f8-b81faf35a0ac kind: Event lastTimestamp: "2026-07-01T21:17:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:19Z" name: sequence-graph-c5814-8549fd89d-pwg26.18be470acfbf96ae namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: d77adc60-40b0-48bd-a894-3b0b82c27efa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c5814-8549fd89d namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 6818e33a-f9d4-4e9d-b6ca-91f9964f2b6c kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: 'Created pod: sequence-graph-c5814-8549fd89d-pwg26' metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814-8549fd89d.18be469729003951 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: b145a61b-538f-4955-8fe2-4f342761bb4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c5814 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 4565951b-7bdd-4910-970a-75859feff9a2 kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: Scaled up replica set sequence-graph-c5814-8549fd89d from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814.18be4697283ee05b namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 31ad7beb-328b-4a53-9e52-99602e8511b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c5814 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 3dbe3714-295f-4024-89c0-7c820147177f kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: InferenceGraph [sequence-graph-c5814] is Ready metadata: creationTimestamp: "2026-07-01T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:37Z" name: sequence-graph-c5814.18be46972d7e62af namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 33581559-7cf8-4a57-941b-aad65b23a948 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5814 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: e9504d8e-2dd1-4cc9-8f42-101ada1ce758 kind: Event lastTimestamp: "2026-07-01T21:08:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:52Z" name: sequence-graph-c5814.18be469aa6ae7d57 namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: 829c8d2a-b2a4-4218-9ff9-a408db59a2b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5814 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: e9504d8e-2dd1-4cc9-8f42-101ada1ce758 kind: Event lastTimestamp: "2026-07-01T21:08:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:52Z" name: sequence-graph-c5814.18be469aa6f52459 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 6abfcf0a-8994-4c2d-ac53-1c152b0352e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:17:42.644319Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-3b58c-b7d74687d-t4jh4 to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be47161483b55c namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 81a9d194-80e7-4fe0-8836-6736242e9ec2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-3b58c-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be47161e693e67 namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 1ece7fa4-ee33-4d2a-9157-cf47b183fa3a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:43Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:43Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:17:43Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be471651a2943e namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: 5196cc0a-29c6-4bee-879c-b2f5b4a7d2d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3b58c} kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:43Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be471653069dbb namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: efead019-4fd9-4063-8bd0-88644daf58cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3b58c} kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:43Z" message: 'Created container: splitter-graph-3b58c' metadata: creationTimestamp: "2026-07-01T21:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:43Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be47165629c656 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 83772961-8d30-476d-a190-6ba51a228a31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3b58c} kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:43Z" message: Started container splitter-graph-3b58c metadata: creationTimestamp: "2026-07-01T21:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:43Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be471656a3192a namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: e0a41801-b40d-4fac-8fd1-a89db179caff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3b58c} kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: Stopping container splitter-graph-3b58c metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:52Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be47186c5f70fe namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 78f3f63b-cb4e-4f7a-89ee-97b1c29b44b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3b58c} kind: Pod name: splitter-graph-3b58c-b7d74687d-t4jh4 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d6a8f4c6-95e2-4d80-a91f-b878dd968cf8 kind: Event lastTimestamp: "2026-07-01T21:18:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:20Z" name: splitter-graph-3b58c-b7d74687d-t4jh4.18be471913050738 namespace: kserve-ci-e2e-test resourceVersion: "27004" uid: c8d82fa5-4f5e-4ab3-a726-7d6169a13575 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-3b58c-b7d74687d namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: b271ae16-6dad-41c5-80e8-dd5c17263007 kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: 'Created pod: splitter-graph-3b58c-b7d74687d-t4jh4' metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c-b7d74687d.18be471613dc7166 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: ccaf9fdc-2b34-4a52-8d02-76a7735779ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 6e8c50ac-aa8a-40ca-a9e6-d133d597324f kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: Scaled up replica set splitter-graph-3b58c-b7d74687d from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c.18be471612b06077 namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: 4aa83677-0160-46a2-bc2b-a3dd4c116a30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 05aa0be1-4846-4f0a-8665-906e7b266c52 kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: InferenceGraph [splitter-graph-3b58c] is Ready metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c.18be47161c7860be namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: 4574a0df-447f-44f9-bade-d479ae8a187f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 05aa0be1-4846-4f0a-8665-906e7b266c52 kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: 'Failed to update status for InferenceGraph "splitter-graph-3b58c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-3b58c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c.18be471621083f37 namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: 8f2ed3d9-5d37-4b59-a723-705fa70437f5 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 05aa0be1-4846-4f0a-8665-906e7b266c52 kind: Event lastTimestamp: "2026-07-01T21:17:42Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-3b58c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:42Z" name: splitter-graph-3b58c.18be4716210892a8 namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: b657e4d6-d916-4f27-89db-d0037e359d50 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:18:52.927194Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6396f-548474fcc8-mxqst to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:18:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:18:52Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be472671b34e78 namespace: kserve-ci-e2e-test resourceVersion: "27287" uid: 8d67e46c-0f3d-4c1b-815a-2d4e8fdddc5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:18:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6396f-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:53Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4726805e64d7 namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: 2ddd515f-87f1-4ec1-b7b7-c4a41ffefb07 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:18:53Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:18:53Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4726aedfe012 namespace: kserve-ci-e2e-test resourceVersion: "27309" uid: b1ed6b86-fcb6-4fc2-b42e-1a0860d7f3b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6396f} kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:18:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:53Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4726b01ab3ce namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: 8e645132-f6cb-49e5-9d03-92c28b38ce9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6396f} kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:18:54Z" message: 'Created container: splitter-graph-6396f' metadata: creationTimestamp: "2026-07-01T21:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:54Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4726b2cd5d42 namespace: kserve-ci-e2e-test resourceVersion: "27312" uid: f9585beb-fa93-4e46-8d1f-16445a1e4bf9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6396f} kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:18:54Z" message: Started container splitter-graph-6396f metadata: creationTimestamp: "2026-07-01T21:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:54Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4726b3404668 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 17932c4c-0e83-4a9b-ab53-49cab891bacf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6396f} kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:27:07Z" message: Stopping container splitter-graph-6396f metadata: creationTimestamp: "2026-07-01T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:07Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be4799a22ccfd3 namespace: kserve-ci-e2e-test resourceVersion: "30756" uid: b9d27ada-c320-4ea3-9def-72f4242f6d80 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6396f} kind: Pod name: splitter-graph-6396f-548474fcc8-mxqst namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 8785dc6c-a28a-43a4-97f9-904212448f0e kind: Event lastTimestamp: "2026-07-01T21:27:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:35Z" name: splitter-graph-6396f-548474fcc8-mxqst.18be479a5cfbeec6 namespace: kserve-ci-e2e-test resourceVersion: "31001" uid: d820934a-2197-44b9-a14f-40e656af677b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6396f-548474fcc8 namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 02c9bcf4-4eef-417e-bf07-9bcb01025e74 kind: Event lastTimestamp: "2026-07-01T21:18:52Z" message: 'Created pod: splitter-graph-6396f-548474fcc8-mxqst' metadata: creationTimestamp: "2026-07-01T21:18:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:52Z" name: splitter-graph-6396f-548474fcc8.18be47267145a71d namespace: kserve-ci-e2e-test resourceVersion: "27284" uid: 5c780c35-22a2-4701-9e17-c12d35ed1e75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6396f namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: c533b44e-5126-4578-a5e6-a6d4e501846d kind: Event lastTimestamp: "2026-07-01T21:18:52Z" message: Scaled up replica set splitter-graph-6396f-548474fcc8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:18:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:52Z" name: splitter-graph-6396f.18be47267056a05a namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: a7446dc7-4953-4db2-b874-c5b79349b243 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6396f namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: 3fdb2db6-3af5-4214-9e69-e4ff2644de2e kind: Event lastTimestamp: "2026-07-01T21:18:53Z" message: InferenceGraph [splitter-graph-6396f] is Ready metadata: creationTimestamp: "2026-07-01T21:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:53Z" name: splitter-graph-6396f.18be47267692ba59 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: e4ff78ba-2135-4f52-a5b8-e7f476b833bd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:19:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6396f namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: e5ebb1f3-bb79-4900-8d1a-622d7c4fabe7 kind: Event lastTimestamp: "2026-07-01T21:19:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:19:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:07Z" name: splitter-graph-6396f.18be4729eef11d4a namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bc9ffff2-d714-4a66-b393-e5f12e103652 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:19:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6396f namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: e5ebb1f3-bb79-4900-8d1a-622d7c4fabe7 kind: Event lastTimestamp: "2026-07-01T21:19:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:19:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:07Z" name: splitter-graph-6396f.18be4729ef5b8fff namespace: kserve-ci-e2e-test resourceVersion: "27418" uid: de3fbe8e-8508-47f6-b954-8301a02dcbbb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:54:32.315444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-25c51-predictor-54665ff85-mxfgx to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:54:32Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45d25e74c32e namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 03faeea8-3b4a-4cfd-b112-155297a094da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:54:32Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45d2781dce3b namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 3e064591-6a8d-4c5a-b18b-242738831948 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:54:32Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45d2799f2421 namespace: kserve-ci-e2e-test resourceVersion: "14369" uid: 92e544b4-3464-4953-9d62-16629d5eb9a4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" in 36.219s (36.219s including waiting). Image size: 1335607390 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45dae873b32e namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: c66925d0-f286-483b-a9f8-678eabfe2539 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45daefec52d5 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: a428c349-5b59-4a76-b013-6524b5e5a956 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45daf22018e3 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: a25a549f-efe1-4130-b044-337d1d34a1fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:09Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45daf22e02fc namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: 3e2ea4d0-c968-4568-a624-ff3f5ede694b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.713s (4.713s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45dc0b209517 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 986e3c4f-35c5-4da6-a09f-bb7ca372a2b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45dc0f0b1850 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: a951f180-f286-4c43-9783-c169ec50ce1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:55:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:13Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45dc0f8a7e2f namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: f1be711c-371a-48d4-9679-21ec37df61f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T20:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:56:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:01Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45dc54474676 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 298d8035-8e24-42af-82d2-67374aeea18f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45ee53a61d25 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 759c0e2f-9a14-41ee-9787-a7b078b5c4a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c51-predictor-54665ff85-mxfgx namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: db188407-051f-40e3-b2f0-8623d40d0429 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-25c51-predictor-54665ff85-mxfgx.18be45ee53a970bc namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 067e6b49-1c63-4c7c-95ff-b528777a6274 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-25c51-predictor-54665ff85 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 50bbf1df-47b8-481f-86e6-9e3cfc2097c1 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: 'Created pod: success-200-isvc-25c51-predictor-54665ff85-mxfgx' metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: success-200-isvc-25c51-predictor-54665ff85.18be45d25e1753af namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 457d214d-a12a-44ad-8777-87a3f74d5870 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: 466fd9fb-7321-450a-9697-2be7d86b63a6 kind: Event lastTimestamp: "2026-07-01T20:54:32Z" message: Scaled up replica set success-200-isvc-25c51-predictor-54665ff85 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:54:32Z" name: success-200-isvc-25c51-predictor.18be45d25ab52d14 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 778fb8eb-37fb-41bd-a672-bcdb014bd451 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: ebeef87c-214d-4b43-835e-c8a32990d80b kind: Event lastTimestamp: "2026-07-01T20:55:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:32Z" name: success-200-isvc-25c51-predictor.18be45d5d93e9059 namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 76dba535-515e-4337-89b7-45550177e3fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: ebeef87c-214d-4b43-835e-c8a32990d80b kind: Event lastTimestamp: "2026-07-01T20:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:32Z" name: success-200-isvc-25c51-predictor.18be45d5d9a76204 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 24f09d94-572d-4bfb-840a-171263498d37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: ebeef87c-214d-4b43-835e-c8a32990d80b kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-25c51-predictor.18be45e3d4fecfc6 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: f996d6e0-5760-405a-a9e5-ec11063ed98f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c51-predictor namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: ebeef87c-214d-4b43-835e-c8a32990d80b kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-25c51-predictor.18be45e3d567858c namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 40d7d2bf-b807-41e2-b480-10dcbdc0fb24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:54:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c51 namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 12abe54d-f88a-47d4-a9e9-35d7aaee4fec kind: Event lastTimestamp: "2026-07-01T20:56:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:11Z" name: success-200-isvc-25c51.18be45d25abbc62d namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 77e3e0a0-08b3-4969-a4d2-51d3e45a954b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c51 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 12abe54d-f88a-47d4-a9e9-35d7aaee4fec kind: Event lastTimestamp: "2026-07-01T20:56:11Z" message: InferenceService [success-200-isvc-25c51] is Ready metadata: creationTimestamp: "2026-07-01T20:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:11Z" name: success-200-isvc-25c51.18be45e9638bf3ca namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 8cd8f519-a511-4ad2-8016-671da01a0f60 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:16:32.673223Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3b58c-predictor-66bc46db55-g6zxg to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be4705c9e993fc namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: b261d861-a170-4aa7-87cf-0fcafa59e0f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3b58c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be4705d30d4582 namespace: kserve-ci-e2e-test resourceVersion: "25394" uid: 44872526-6b7a-4e84-a156-d6e0c4661b54 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be4706072a30a6 namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: d303a8be-19d4-477c-b8d7-9996495798cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be470608a326f4 namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: ef684a41-5979-423c-98e2-21a65a02f36b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47060ba23366 namespace: kserve-ci-e2e-test resourceVersion: "25429" uid: 5b7c4f58-1eea-4b44-9193-857f4e607bc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47060c113dc0 namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 04ad619e-a7f2-4a1c-8e1a-8912e9b6505c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47060c1a9cdb namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 688e06e3-87be-4d42-b482-993f69669858 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47060eee4808 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 8b3c4acb-462b-4b62-ac61-a7aec81ca1a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:16:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:33Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47060f5e4c73 namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: 6ed2bb9f-1088-43f9-bf00-fe2ada01873e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:17:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:20Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be470628fed543 namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 97943c5c-4c3a-4084-bd5e-10b4bd5ed91f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be4718743cccc5 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 191732e6-fc29-49ab-9f0e-86b029e11e18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be4718743dd1bd namespace: kserve-ci-e2e-test resourceVersion: "26653" uid: 91999358-cb51-423b-b07a-25ea3b659e4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 08d15b65-39b4-4b32-8c82-a4535bba8ee1 kind: Event lastTimestamp: "2026-07-01T21:17:55Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:55Z" name: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg.18be47190531aa2b namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: 1afe96fd-bd3f-46e0-9384-d7d60bf3586a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3b58c-predictor-66bc46db55 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: fad54c63-10e7-4c9d-bb1a-f93bd2bca79d kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: 'Created pod: success-200-isvc-3b58c-predictor-66bc46db55-g6zxg' metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-3b58c-predictor-66bc46db55.18be4705c99ea2d2 namespace: kserve-ci-e2e-test resourceVersion: "25351" uid: e98d08fb-fdb0-4e0f-9c2c-0f980b6fb906 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 8a2c9126-6e2d-423a-ace2-6675f3ce0a65 kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Scaled up replica set success-200-isvc-3b58c-predictor-66bc46db55 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-3b58c-predictor.18be4705c4959ce4 namespace: kserve-ci-e2e-test resourceVersion: "25329" uid: 9b1e9502-5829-4ea1-bb5b-e6ad00098e42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:16:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 66723751-16d5-4099-aafa-6d17d844892a kind: Event lastTimestamp: "2026-07-01T21:17:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:17Z" name: success-200-isvc-3b58c-predictor.18be4709430c0caf namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 5838bb8b-b3e4-47fc-ad6f-dbcbc532d844 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:16:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 66723751-16d5-4099-aafa-6d17d844892a kind: Event lastTimestamp: "2026-07-01T21:17:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:17Z" name: success-200-isvc-3b58c-predictor.18be47094376d36c namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 6fb726ae-d403-45aa-b5e8-368317b4667d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 66723751-16d5-4099-aafa-6d17d844892a kind: Event lastTimestamp: "2026-07-01T21:17:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:47Z" name: success-200-isvc-3b58c-predictor.18be4713c0e40b67 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: afa7b3ce-591d-490c-821d-4d94b7d504d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b58c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 66723751-16d5-4099-aafa-6d17d844892a kind: Event lastTimestamp: "2026-07-01T21:17:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:47Z" name: success-200-isvc-3b58c-predictor.18be4713c1596d0c namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: c676d4df-0c0c-495e-86cf-b19ff448b50e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3b58c namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: e894b235-6930-42cb-95fd-a02a48980aa9 kind: Event lastTimestamp: "2026-07-01T21:17:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:30Z" name: success-200-isvc-3b58c.18be4705c483501e namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: f2fbe32b-f439-4d5e-81c4-8de1667fa341 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3b58c namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: e894b235-6930-42cb-95fd-a02a48980aa9 kind: Event lastTimestamp: "2026-07-01T21:17:30Z" message: InferenceService [success-200-isvc-3b58c] is Ready metadata: creationTimestamp: "2026-07-01T21:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:30Z" name: success-200-isvc-3b58c.18be471338aa2d79 namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: d66d37f6-d591-4a2c-a296-68192f16e0a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:17:52.855859Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26652" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6396f-predictor-85b9f855cc-l99gt to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718752c9514 namespace: kserve-ci-e2e-test resourceVersion: "26657" uid: ff7ac7ad-ceb2-4e9a-bc91-6cfbbbe99086 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6396f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be47187e939599 namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: 199b8092-fde1-47b6-a989-9aab01bf4162 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718b2320268 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 7229ea0a-525d-445f-8f83-92b22d32fe3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718b3dbb6bb namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: 9024b8e6-b37e-45e4-b822-fc656242011d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718b812cecf namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 5178d291-90f5-4373-98f2-69d1ff08fe82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718b88b653e namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 9cd21e82-493a-46ec-8eaf-4b67eee23ab4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:53Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718b8958aba namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: 1bfa234e-ac9b-4123-8804-987970535068 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718bbaef374 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: 0a3c2010-5cef-4dfe-9f1c-01af3b699272 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:17:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:54Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4718bc22f9a7 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: b6d0747d-7e5e-4f4e-ae8c-bc1b780bfb6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:18:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:31Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be47191ab3c8b4 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: ab45e614-9b45-4eba-923d-6745127e650b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:27:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:07Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4799acc4d15f namespace: kserve-ci-e2e-test resourceVersion: "30774" uid: 88386d4e-2e02-4c54-b963-9a55468a8283 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 50527622-67cb-46d0-a3d1-95ab459aaae0 kind: Event lastTimestamp: "2026-07-01T21:27:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:27:07Z" name: success-200-isvc-6396f-predictor-85b9f855cc-l99gt.18be4799acc573a7 namespace: kserve-ci-e2e-test resourceVersion: "30775" uid: 8db3b176-2bf2-4d75-b0a6-f447fd7a2752 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6396f-predictor-85b9f855cc namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 55f56781-1e9c-4515-8351-b734afd0a817 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'Created pod: success-200-isvc-6396f-predictor-85b9f855cc-l99gt' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-6396f-predictor-85b9f855cc.18be471874c074b5 namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: 2503389f-f2da-4884-94ee-18e50e8a64b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: ce87722b-d6a4-40d9-876b-c876cff2ace2 kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: Scaled up replica set success-200-isvc-6396f-predictor-85b9f855cc from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-6396f-predictor.18be4718729e523c namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 9660b186-49e6-437f-8237-e2467d66f976 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 2bc6056c-2d6a-483e-94c6-ae8383bd64b1 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: success-200-isvc-6396f-predictor.18be471bf1312ede namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: f422143f-35c7-4e9e-b6c3-59000c352910 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 2bc6056c-2d6a-483e-94c6-ae8383bd64b1 kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: success-200-isvc-6396f-predictor.18be471bf1726206 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 8c346b7a-20e2-417b-90f8-484be3dd28c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 2bc6056c-2d6a-483e-94c6-ae8383bd64b1 kind: Event lastTimestamp: "2026-07-01T21:19:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:07Z" name: success-200-isvc-6396f-predictor.18be4722eef8c44b namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: 59ce0c1b-7cc7-46ef-bb48-bc17406b965f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6396f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 2bc6056c-2d6a-483e-94c6-ae8383bd64b1 kind: Event lastTimestamp: "2026-07-01T21:19:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:19:07Z" name: success-200-isvc-6396f-predictor.18be4722ef6c0d2b namespace: kserve-ci-e2e-test resourceVersion: "27415" uid: f9eade8d-8ef6-4324-967e-0d980caa7674 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 361f9ee7-cc5e-4f2b-87cc-6f04e2af126f kind: Event lastTimestamp: "2026-07-01T21:18:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:41Z" name: success-200-isvc-6396f.18be471872af53eb namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: 718d79f8-d9e9-4c82-9862-291b8f32e7df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 361f9ee7-cc5e-4f2b-87cc-6f04e2af126f kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-6396f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6396f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-6396f.18be4718738c0588 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 9bf37966-4908-4908-818e-1b08f4935cee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 361f9ee7-cc5e-4f2b-87cc-6f04e2af126f kind: Event lastTimestamp: "2026-07-01T21:17:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6396f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:52Z" name: success-200-isvc-6396f.18be4718738c3ecb namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: cab5fb8d-e0e3-44b2-b921-2880a6a21daa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6396f namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 361f9ee7-cc5e-4f2b-87cc-6f04e2af126f kind: Event lastTimestamp: "2026-07-01T21:18:41Z" message: InferenceService [success-200-isvc-6396f] is Ready metadata: creationTimestamp: "2026-07-01T21:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:41Z" name: success-200-isvc-6396f.18be4723d98c72ba namespace: kserve-ci-e2e-test resourceVersion: "27192" uid: af01a09c-bcf2-45c5-a141-ea8f5cfe9819 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:06:17.199205Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46767ccd5ef7 namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 3ff53aac-7280-4164-a375-bbd5af75c66e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be467696bec598 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 17c80615-2700-4471-9104-febfb3d7a53d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769834d0cb namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: fcec572e-77bc-43fa-bb34-eafda5e61560 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769bbe1531 namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 1a4d0e49-bed7-48a9-a7ce-c81b3bb7a7ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769c464c90 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 61e25ab2-510e-4a90-a82e-defdaceef534 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769c51eb98 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: f0c48ddf-a1d4-4f26-bb8f-bd3ff78f39f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769f10a4f2 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: c03c5810-0818-4abf-9a9d-4120de090a4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46769f8466b8 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 9ba0617f-5454-4f7d-8152-ac409f32cd19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:07:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:03Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be4676a000e62a namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 1841318e-f16e-4cb7-898e-993cd19dcc43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46892bf9353c namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 8cc67897-7961-4abe-bc86-df5e04c46a2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46892bf99d76 namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 93e36b89-815b-4ed6-9468-dec55dadd97c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c74f80f2-e534-4b04-af47-e3425926839f kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v.18be46897cae84f8 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 48fea9ae-6458-4b7e-818c-95b5cad82ee7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7cf3f-predictor-5899db67f5 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: f0a5a074-3493-4b1d-93e4-a658b35a91a7 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Created pod: success-200-isvc-7cf3f-predictor-5899db67f5-kbg5v' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor-5899db67f5.18be46767c01630b namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: fb4bd511-0d05-4f63-8ee9-e6dafd818a31 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: f4460d4c-55c4-4df2-93b9-61096c5395f2 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Scaled up replica set success-200-isvc-7cf3f-predictor-5899db67f5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f-predictor.18be46767a0eb8ee namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: b9061d0c-c5ea-4a4f-81b1-b0010b7d5d53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 50d4fce6-9d5c-4743-bec2-d41461198fa7 kind: Event lastTimestamp: "2026-07-01T21:07:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:02Z" name: success-200-isvc-7cf3f-predictor.18be4679f8ddadb0 namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: ee625cb7-4be5-4903-9ba7-2b8fdb2bb538 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 50d4fce6-9d5c-4743-bec2-d41461198fa7 kind: Event lastTimestamp: "2026-07-01T21:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:02Z" name: success-200-isvc-7cf3f-predictor.18be4679f93ceaf4 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 5b8810da-e05e-405f-b0dd-cb024f62bfd7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: 50d4fce6-9d5c-4743-bec2-d41461198fa7 kind: Event lastTimestamp: "2026-07-01T21:07:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:32Z" name: success-200-isvc-7cf3f-predictor.18be468475db29f6 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: c41b3bfd-41ac-468c-97ee-b8082849326d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7cf3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: 50d4fce6-9d5c-4743-bec2-d41461198fa7 kind: Event lastTimestamp: "2026-07-01T21:07:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:32Z" name: success-200-isvc-7cf3f-predictor.18be4684764bc21c namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: e116c022-f927-49f2-8afd-649228d1f897 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: e6bceb4a-e164-4b35-8c20-4652e232240b kind: Event lastTimestamp: "2026-07-01T21:07:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:13Z" name: success-200-isvc-7cf3f.18be46767a03272b namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 995769be-2a5e-4dd1-b7a6-e33324ef30e9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: e6bceb4a-e164-4b35-8c20-4652e232240b kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-7cf3f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7cf3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f.18be46767a7459ca namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: cd683879-7a01-4708-ba0d-d6cc21025a54 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: e6bceb4a-e164-4b35-8c20-4652e232240b kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7cf3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-7cf3f.18be46767a74b521 namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 7dd73e1d-39bf-4b36-8c85-f81754642e3d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7cf3f namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: e6bceb4a-e164-4b35-8c20-4652e232240b kind: Event lastTimestamp: "2026-07-01T21:07:13Z" message: InferenceService [success-200-isvc-7cf3f] is Ready metadata: creationTimestamp: "2026-07-01T21:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:13Z" name: success-200-isvc-7cf3f.18be4683afd5bcf4 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 89a4d29a-2c25-44ba-ae97-1123a9f0ffc7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:17.581130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-86dc6-predictor-6758b686f5-zdnwm to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be46848bd872e3 namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 38d80ee0-0c6c-4665-9582-ce71e08ec6a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684a53de438 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: a118ccc4-f885-4c9b-8108-a8218d80a336 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684a6990e8c namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 97bbdd8d-2448-48b7-b83a-8e619a231fd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684a9770b57 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 0cd90be7-6cf1-44bd-a1d5-ef160ad1f962 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684a9e233ec namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 611b778c-90e4-4b68-bb02-2905dfd21582 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684a9ea8b85 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: ef48a4a6-f213-4ae8-ab23-3574813b135f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684ad122c63 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: ec914c1e-4f1d-492b-9d71-fa3edd275b65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:07:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:18Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684ad810dc1 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: d06978fa-e4d9-4fc1-bb12-156eee6ca9b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:16:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:35Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4684e45b7164 namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: fcdad211-9850-429c-b049-299a985d40f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4705c9b6e6a9 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: 0c333f22-57dc-4418-8e55-dc3969db7ee8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:16:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:32Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be4705c9b7741b namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: ef733bd4-8f11-4e02-a1a8-0c6bc8e8e0fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: a591d25f-cdbd-4368-b760-6b9a71a763dc kind: Event lastTimestamp: "2026-07-01T21:16:35Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:35Z" name: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm.18be470658c8e52e namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 09125717-d31f-4c7b-a6eb-a779abf19119 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-86dc6-predictor-6758b686f5 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: e7816db4-8305-44ce-84e8-7a2a325237e3 kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'Created pod: success-200-isvc-86dc6-predictor-6758b686f5-zdnwm' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-86dc6-predictor-6758b686f5.18be46848b738fe3 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: dc39143d-9d66-4d4f-b945-a8e86cbde6b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 6be534d1-fd5d-45aa-b727-96b7f4ef9b5b kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Scaled up replica set success-200-isvc-86dc6-predictor-6758b686f5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-86dc6-predictor.18be46848988aabb namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 58a7135f-77bd-4595-a839-81bc64e4092d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:07:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: b5ce0ace-d8b5-48af-af18-54e6f3ebf565 kind: Event lastTimestamp: "2026-07-01T21:08:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:02Z" name: success-200-isvc-86dc6-predictor.18be46880805bd92 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 125207d3-6ace-4f13-8b2b-66fde12817e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:07:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: b5ce0ace-d8b5-48af-af18-54e6f3ebf565 kind: Event lastTimestamp: "2026-07-01T21:08:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:02Z" name: success-200-isvc-86dc6-predictor.18be468808667b56 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 42e9e47c-d27c-4464-9a46-38b8fd296909 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: b5ce0ace-d8b5-48af-af18-54e6f3ebf565 kind: Event lastTimestamp: "2026-07-01T21:08:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:47Z" name: success-200-isvc-86dc6-predictor.18be469285048aa3 namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: c3212443-be9e-424a-a2e0-c57bde973334 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86dc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: b5ce0ace-d8b5-48af-af18-54e6f3ebf565 kind: Event lastTimestamp: "2026-07-01T21:08:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:47Z" name: success-200-isvc-86dc6-predictor.18be469285456b72 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 890e4f04-a35f-42eb-9098-b07f0e72e2ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: c1505e43-00d4-4244-8f64-f462c6a9a6d1 kind: Event lastTimestamp: "2026-07-01T21:08:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:05Z" name: success-200-isvc-86dc6.18be468489a0d9ba namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 5ed04ae8-b81e-404b-a793-0403a023871f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: c1505e43-00d4-4244-8f64-f462c6a9a6d1 kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-86dc6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-86dc6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-86dc6.18be46848a8dd7d1 namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: a283eeea-b907-4fdc-b218-34a7a80b56ae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: c1505e43-00d4-4244-8f64-f462c6a9a6d1 kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-86dc6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-86dc6.18be46848a8e1dfe namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 6522b136-295e-45e3-a80f-1dbc09079fb4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86dc6 namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: c1505e43-00d4-4244-8f64-f462c6a9a6d1 kind: Event lastTimestamp: "2026-07-01T21:08:05Z" message: InferenceService [success-200-isvc-86dc6] is Ready metadata: creationTimestamp: "2026-07-01T21:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:05Z" name: success-200-isvc-86dc6.18be468fa247fa1a namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 14658917-6e2d-46b8-8256-adc122a219d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:57:02.366016Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a93e1-predictor-657768547-67mn2 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f54e2adae3 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 80e0fc3c-e81a-42ef-9ecf-e5f55d05ba18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f568948c65 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 78a0a115-7952-4ec9-9c89-67243ff4b444 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f56a37c788 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 6d2a3122-2741-4a08-848f-aadc4d0760b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f56f1ee244 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: e7808fd7-dcd3-4cce-bcf7-62a2a491dcc1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f56f991ab8 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 2ead0261-f8e2-4d68-901b-4abaaf0ca93d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f56fa27ce8 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 75124cf3-a90e-45e5-9685-aeded897a5f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f572fc3855 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: b8a4f900-07a2-4f74-a105-59da74d25ec8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f573737781 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 10f00a3f-5db9-4569-8b75-6999d125b2c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T20:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T21:06:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:20Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be45f5d2356308 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: a5a6b511-0037-4f95-b4a7-5510a892745b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be46768143ae82 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 249764b3-3634-42a9-a658-6024d6ba2d0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T21:06:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:17Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be467681443136 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: 72ae38e8-c9cc-46e4-99e0-972ecce0afb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93e1-predictor-657768547-67mn2 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: e1392569-add7-437e-b0d7-ef1ad3b24949 kind: Event lastTimestamp: "2026-07-01T21:06:20Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:20Z" name: success-200-isvc-a93e1-predictor-657768547-67mn2.18be467746a17071 namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: aa202319-2078-47ad-9b2e-5bf116f98350 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a93e1-predictor-657768547 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 43e1a280-a22b-4726-b872-78179091a93a kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'Created pod: success-200-isvc-a93e1-predictor-657768547-67mn2' metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor-657768547.18be45f54d9d55c0 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 519fd403-8d5c-4c5f-b2b4-db5fcfea6004 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: eb17efa6-bdde-4f51-a30d-7c43bcd9ff78 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: Scaled up replica set success-200-isvc-a93e1-predictor-657768547 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-a93e1-predictor.18be45f54b47762a namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 6dd143db-974a-40ae-9887-d7cd9a8b67df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 9ddbb63a-826f-4b3f-a021-9c06e41d8cac kind: Event lastTimestamp: "2026-07-01T20:57:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:17Z" name: success-200-isvc-a93e1-predictor.18be45f8c9c45fe3 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 1b585f2a-d29c-4b3e-a52e-805e97881b33 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 9ddbb63a-826f-4b3f-a021-9c06e41d8cac kind: Event lastTimestamp: "2026-07-01T20:57:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:17Z" name: success-200-isvc-a93e1-predictor.18be45f8ca258839 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: f1cd1bdc-4d56-4192-bb63-c896d00e2ef0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 9ddbb63a-826f-4b3f-a021-9c06e41d8cac kind: Event lastTimestamp: "2026-07-01T20:58:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:32Z" name: success-200-isvc-a93e1-predictor.18be45fc49971160 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 671cc973-d565-4564-8f33-dc5a5cec6327 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 9ddbb63a-826f-4b3f-a021-9c06e41d8cac kind: Event lastTimestamp: "2026-07-01T20:58:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:32Z" name: success-200-isvc-a93e1-predictor.18be45fc4a08cdd6 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 1e8446db-c527-46f8-8539-9403285ddab8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 13e940c5-dd87-42b6-bbab-75c7e4601dfe kind: Event lastTimestamp: "2026-07-01T20:57:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:50Z" name: success-200-isvc-a93e1.18be45f54b7f05b3 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: b2430836-61d4-4455-8940-bbb33020fcaa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93e1 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 13e940c5-dd87-42b6-bbab-75c7e4601dfe kind: Event lastTimestamp: "2026-07-01T20:57:50Z" message: InferenceService [success-200-isvc-a93e1] is Ready metadata: creationTimestamp: "2026-07-01T20:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:50Z" name: success-200-isvc-a93e1.18be46008d44ebaf namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 67f199e6-27f7-4047-b410-3b333d2db7cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:37.440814Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c5814-predictor-84c478fd7-87szc to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46892b9319e6 namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 006657fe-5bf1-42c5-8fdb-560d1111ba5f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c5814-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46893be73453 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 2cd8a6c8-32d4-4faf-af5a-8705af6254ce reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be4689694ccbcc namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 78ad8c2c-e75f-43fc-b147-eba00b546da2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46896b23a9bc namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 439de916-235b-42ec-a461-e480411e559f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46896e8fcb3e namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: 87ca4bd2-7d5e-43ae-996a-6b68586c99e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46896f42ac1b namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 033759a8-c930-4b63-9253-29b66b9a5453 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be46896f4ba7c1 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 1415ad32-e4ea-4e18-9e27-8ae9846d5334 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be4689723d6a3c namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: b241112a-fae4-4751-b681-e82dd0041875 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:38Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be468972ad91a0 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 99c000d0-aa30-4868-a123-fef67acbfe75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:16:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:55Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be468992805241 namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: ec96889f-8305-4f6d-b922-ac3d5d1790d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be470a5ac59cc1 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 3a08a84f-fb65-4724-8a8e-5efe9ba98e74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be470a5ac62fb1 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: e831c067-5ee5-4927-a7bf-5592bb78ed3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5814-predictor-84c478fd7-87szc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: cdfc78e7-3714-46d2-b056-5feb3f503126 kind: Event lastTimestamp: "2026-07-01T21:16:55Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:55Z" name: success-200-isvc-c5814-predictor-84c478fd7-87szc.18be470b06df809b namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: e29abd47-931d-48e1-b945-4fc9712e74e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c5814-predictor-84c478fd7 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 7eea73cb-d440-4959-a892-69597c1110c8 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'Created pod: success-200-isvc-c5814-predictor-84c478fd7-87szc' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-c5814-predictor-84c478fd7.18be46892b0e3c7b namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 66d3eda6-82b2-4d22-9361-75d846f219c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 7b59ab82-78e0-4077-bc5e-61d058fbfa62 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: Scaled up replica set success-200-isvc-c5814-predictor-84c478fd7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:37Z" name: success-200-isvc-c5814-predictor.18be468928d2e507 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 07bf6a65-9f47-488d-9539-44b800d8ba21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 9f9226ae-5470-4768-ae74-cc96bba63dd2 kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:37Z" name: success-200-isvc-c5814-predictor.18be468ca7943acd namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 46cbe04c-a2cb-402d-b90a-edc9aed42575 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 9f9226ae-5470-4768-ae74-cc96bba63dd2 kind: Event lastTimestamp: "2026-07-01T21:08:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:37Z" name: success-200-isvc-c5814-predictor.18be468ca7d8d94a namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: 007a66a9-dc9f-405e-99a6-8bc31117ff91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 9f9226ae-5470-4768-ae74-cc96bba63dd2 kind: Event lastTimestamp: "2026-07-01T21:08:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:07Z" name: success-200-isvc-c5814-predictor.18be469026915047 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: 15c55941-4bd1-434d-86b0-3fdf42cc9c4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5814-predictor namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 9f9226ae-5470-4768-ae74-cc96bba63dd2 kind: Event lastTimestamp: "2026-07-01T21:08:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:07Z" name: success-200-isvc-c5814-predictor.18be46902706ca1a namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 9da1eac3-50bc-4545-aac6-39394dbf1e47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 87668731-f5db-4e21-8e4f-62edf6056ae7 kind: Event lastTimestamp: "2026-07-01T21:08:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:25Z" name: success-200-isvc-c5814.18be46892913fe15 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 5d9ac5cd-d50f-4f4e-b2f0-d2d0b6666212 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:08:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5814 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 87668731-f5db-4e21-8e4f-62edf6056ae7 kind: Event lastTimestamp: "2026-07-01T21:08:25Z" message: InferenceService [success-200-isvc-c5814] is Ready metadata: creationTimestamp: "2026-07-01T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:08:25Z" name: success-200-isvc-c5814.18be469450cb7412 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 3ae44638-84cd-455b-8a9f-1e81a63585a0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:16:52.290635Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a5b338b6e namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 16ce5ca3-aa90-4aa1-a5b2-e53fd6d1471f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a75a48eed namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: c23780b2-60ff-4ab2-83b0-1e28b86908fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a773b858e namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 108613be-4a66-4e5a-80b0-0bdf39f3419a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a7a57ee83 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 7ea1fa6b-7c3f-41fb-adb0-57a51a92a616 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a7aec1c3f namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: 1b828d5a-df1c-40c3-bf1a-3bdbd22a3a8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a7af7f16a namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 59653b66-5296-41b6-80e0-7c86b9f1b599 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a7de2b52f namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 8a00072d-1636-4025-a8f8-d64b254c0d2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a7e576ba7 namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: f7ff503c-9ef4-4fd5-a250-6097d46df277 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:17:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:17:39Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be470a9aa8feec namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: 0650ed21-8262-4750-ad42-2d268bb52986 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:34:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:21Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be47feae4494e1 namespace: kserve-ci-e2e-test resourceVersion: "33793" uid: b3bd489c-ff69-428f-836f-695bd70db935 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:34:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:21Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be47feae451b0b namespace: kserve-ci-e2e-test resourceVersion: "33794" uid: c4c236f5-da84-423f-8f45-5513e3495618 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 8475069f-ef34-4979-9c40-9dcb2723997b kind: Event lastTimestamp: "2026-07-01T21:34:24Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:24Z" name: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2.18be47ff4f95ddcb namespace: kserve-ci-e2e-test resourceVersion: "33865" uid: ffd31f0d-86b2-4ab8-bf59-fe8f3ba0e181 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ce993-predictor-6d75865fd7 namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 84f7b7d1-a5bd-4a47-9b6b-5c8c385082a5 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: 'Created pod: success-200-isvc-ce993-predictor-6d75865fd7-pt9r2' metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor-6d75865fd7.18be470a5bc66887 namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: 31ab7058-44d0-4eee-a076-b2a3622477c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: e6597f44-37c3-461b-b4b3-73afe32ee288 kind: Event lastTimestamp: "2026-07-01T21:16:52Z" message: Scaled up replica set success-200-isvc-ce993-predictor-6d75865fd7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:16:52Z" name: success-200-isvc-ce993-predictor.18be470a58d5930a namespace: kserve-ci-e2e-test resourceVersion: "25898" uid: 9a5d927e-0b16-437d-a339-ae90f1c04632 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 4883e47c-c406-4eae-a6de-a4661f83c09c kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: success-200-isvc-ce993-predictor.18be470dd7644c07 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 9f2eca49-3631-4219-9637-f68e2f1cfb46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce993-predictor namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 4883e47c-c406-4eae-a6de-a4661f83c09c kind: Event lastTimestamp: "2026-07-01T21:18:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:22Z" name: success-200-isvc-ce993-predictor.18be470dd7cedf79 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: 03618222-51d7-4e9a-b4fc-30152c0d2226 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 708c8d64-b973-438b-8a53-1244f8b9df57 kind: Event lastTimestamp: "2026-07-01T21:17:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:49Z" name: success-200-isvc-ce993.18be470a59049168 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 57adedb2-4f7f-4733-816b-30e76e967681 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:17:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 708c8d64-b973-438b-8a53-1244f8b9df57 kind: Event lastTimestamp: "2026-07-01T21:17:49Z" message: InferenceService [success-200-isvc-ce993] is Ready metadata: creationTimestamp: "2026-07-01T21:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:17:49Z" name: success-200-isvc-ce993.18be4717aaad85bd namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 5d2010e3-a36f-4501-813e-72ac7e75ad22 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:05:57.343315Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671dd4c7992 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 2431033d-8919-430d-860b-a46146a50955 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671f6f14aa6 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 3142d0b8-78f1-4d01-b3cf-d4a346d0e54f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671f84ed331 namespace: kserve-ci-e2e-test resourceVersion: "20069" uid: 82269c6c-9441-42f3-aeb2-37452ccd6b54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671fb717610 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 138c3b78-3f6f-4c93-96ac-062967d44f29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671fbe0c01a namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 83d5cb18-d2f1-4ec2-a431-7e87c40f4078 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671fbea8f8e namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: a32c5d5c-cef0-40fe-9310-e0872b458528 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671fe8760b5 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 91f533ff-d59a-4bcf-99e6-d160c528d50b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be4671ff2a96b7 namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 7e7a5550-f9fa-4668-a268-a9642ba234fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T21:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:06:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:45Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be46722e90633c namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 0ea1a752-8467-4d36-9b42-52915dec462f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be46848b02b042 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 6e1efcd6-3209-422d-99e5-fedca2cf8ce7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:07:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:17Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be46848b032aaa namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 2e161e5c-f4f5-4d0e-8d76-11be5d57a3aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0f7441a7-bb69-4d68-8073-98c87a63a34d kind: Event lastTimestamp: "2026-07-01T21:07:20Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:20Z" name: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7.18be468546e4e7d1 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 3bf8ee07-9b0e-4b1a-9892-c1f90f85cce4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f1d61-predictor-69d9f6659c namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 9644458f-5fb1-4670-9fba-0db455cdbede kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Created pod: success-200-isvc-f1d61-predictor-69d9f6659c-rznj7' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor-69d9f6659c.18be4671dce5960d namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: faa83a44-83cb-4d7e-902e-d4dbe4257adf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 35f54a2d-4a3c-4126-81ca-f65aa1dfd858 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Scaled up replica set success-200-isvc-f1d61-predictor-69d9f6659c from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61-predictor.18be4671d90204dd namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 18ea0b3b-7230-4a5b-bb56-0c99018b8f02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: bd0ea55c-a63f-4145-9832-310be7a8aa15 kind: Event lastTimestamp: "2026-07-01T21:06:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:27Z" name: success-200-isvc-f1d61-predictor.18be46755798b69d namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 1424c329-9c37-4167-b7c4-188ccbfa5584 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: bd0ea55c-a63f-4145-9832-310be7a8aa15 kind: Event lastTimestamp: "2026-07-01T21:06:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:27Z" name: success-200-isvc-f1d61-predictor.18be46755803303d namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: f3366e76-8681-4161-a2ff-988e99118de2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: bd0ea55c-a63f-4145-9832-310be7a8aa15 kind: Event lastTimestamp: "2026-07-01T21:07:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:12Z" name: success-200-isvc-f1d61-predictor.18be467c559ab881 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 77fcb9b4-1ced-4c26-8a39-c60a0d04c1d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d61-predictor namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: bd0ea55c-a63f-4145-9832-310be7a8aa15 kind: Event lastTimestamp: "2026-07-01T21:07:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:12Z" name: success-200-isvc-f1d61-predictor.18be467c55e51d6d namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: b7671525-4672-48ae-baff-98067e2acb47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 958cd480-e1d3-44e0-97ce-8ece4bc27c65 kind: Event lastTimestamp: "2026-07-01T21:06:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:55Z" name: success-200-isvc-f1d61.18be4671d90ea641 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 63c0a679-ce53-496f-9b27-2be457099eea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 958cd480-e1d3-44e0-97ce-8ece4bc27c65 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Failed to update status for InferenceService "success-200-isvc-f1d61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61.18be4671d9a7ef4d namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 83781d44-dc36-4cc4-aae7-bf17dccac31d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 958cd480-e1d3-44e0-97ce-8ece4bc27c65 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f1d61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f1d61.18be4671d9a83444 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: 1ca91897-210a-4f55-af71-4974d0c6264b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d61 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 958cd480-e1d3-44e0-97ce-8ece4bc27c65 kind: Event lastTimestamp: "2026-07-01T21:06:55Z" message: InferenceService [success-200-isvc-f1d61] is Ready metadata: creationTimestamp: "2026-07-01T21:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:06:55Z" name: success-200-isvc-f1d61.18be467f79b18680 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 1d6c6e1d-9dda-40cb-bd7d-8cc67807f152 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:56:32.367072Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f347e-predictor-9c7897d59-775b4 to ip-10-0-131-198.ec2.internal metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee52173719 namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 836f193f-48ce-446e-bd17-f569f608030c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee6bc82bf3 namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: cab4c192-71ef-4f68-88f5-353381e56767 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee6e27e9b0 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 150a739c-2ce6-4de4-aa0d-353ba936b92b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee71e9d4aa namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 7cccd1f3-5cba-4533-8f14-333dc429d55e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee725bac2a namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 9dc0f2dc-b109-4b15-8cdd-35c73fd6d5b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee72681451 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 545a5e8c-910f-4228-ad1b-8d42a01abb5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee757de92e namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 1663c67e-1099-46ee-b29a-b0d6ea5c1d45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee75e547f0 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: a082fea9-57d3-49e5-b0f1-8565221c2f62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T20:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T21:05:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:59Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be45ee916d2ea2 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 2b919a81-76cb-4d57-8e6e-abb8bcb8e9d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be4671de56f65d namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: acb19739-4617-4c21-b430-8cd2d0253191 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be4671de57ff67 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 4b1dc107-89ee-4ee6-8777-2b520a75c90a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f347e-predictor-9c7897d59-775b4 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 447b6d3c-1be1-4053-91db-e8f11d74445f kind: Event lastTimestamp: "2026-07-01T21:05:59Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:59Z" name: success-200-isvc-f347e-predictor-9c7897d59-775b4.18be467259f18905 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: ece549c9-f456-40d5-9813-3b151f7ec7d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-198.ec2.internal source: component: kubelet host: ip-10-0-131-198.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f347e-predictor-9c7897d59 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 905bc642-eb32-4cf0-a9fd-40ccdaeb717e kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Created pod: success-200-isvc-f347e-predictor-9c7897d59-775b4' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor-9c7897d59.18be45ee517a3ea1 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 3191c473-e443-49fd-8146-30ddedd7e873 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: bb083064-acc7-47bb-bfee-b7673dbce9fd kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Scaled up replica set success-200-isvc-f347e-predictor-9c7897d59 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e-predictor.18be45ee4e6c726d namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 12e47bc7-0a99-455a-b035-7be1047ea49f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: ae79eb08-9cb1-4010-9a8b-3185f5987a53 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-f347e-predictor.18be45f1ccf3e17e namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 204894fa-bc78-420d-93ba-bd8fdcc75bb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: ae79eb08-9cb1-4010-9a8b-3185f5987a53 kind: Event lastTimestamp: "2026-07-01T20:57:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:02Z" name: success-200-isvc-f347e-predictor.18be45f1cd58edf1 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 43444671-d2b2-4652-b0a9-375f3f04e5d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: ae79eb08-9cb1-4010-9a8b-3185f5987a53 kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: success-200-isvc-f347e-predictor.18be45f8cb1eb9a3 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 1159a31e-7ec1-46c0-8137-bdb1f3b2ef29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f347e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: ae79eb08-9cb1-4010-9a8b-3185f5987a53 kind: Event lastTimestamp: "2026-07-01T20:57:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:47Z" name: success-200-isvc-f347e-predictor.18be45f8cb84540c namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: ce76deec-637a-4112-8c3c-380c8b83c54b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: fee97de2-c98b-418d-8c0b-40cd08d5f7a9 kind: Event lastTimestamp: "2026-07-01T20:57:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:29Z" name: success-200-isvc-f347e.18be45ee4e73f268 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: c456b806-b69d-4197-8358-7a604a3cf238 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: fee97de2-c98b-418d-8c0b-40cd08d5f7a9 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-f347e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f347e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e.18be45ee4efdef0a namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 9990e193-ce2a-47fe-b021-e1879358dad4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: fee97de2-c98b-418d-8c0b-40cd08d5f7a9 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f347e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:32Z" name: success-200-isvc-f347e.18be45ee4efe48ff namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 9b87cfba-3cbd-45f6-be96-5aa62ebc941a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f347e namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: fee97de2-c98b-418d-8c0b-40cd08d5f7a9 kind: Event lastTimestamp: "2026-07-01T20:57:29Z" message: InferenceService [success-200-isvc-f347e] is Ready metadata: creationTimestamp: "2026-07-01T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:29Z" name: success-200-isvc-f347e.18be45fba0b233a7 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: dbf56437-8cad-4ad6-834c-53f5ab5659ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:56:22.051443Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-25c51-56c67f6cb4-wlzqd to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ebeb3b347c namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 2edbe89a-5b37-484d-af74-2451e7274bff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ec04c2221c namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 4a5ae4dc-bdf8-4da3-9156-c12abdd662f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ec05fb521d namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 797dc411-1eb5-48aa-ad47-9fbcb8147b05 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" in 2.204s (2.205s including waiting). Image size: 218070105 bytes.' metadata: creationTimestamp: "2026-07-01T20:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:24Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ec8969b2b7 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: a81f6adc-1d77-408c-8265-bc6e9730bdf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:24Z" message: 'Created container: switch-graph-25c51' metadata: creationTimestamp: "2026-07-01T20:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:24Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ec8c55cf5d namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 1fd49493-88ca-4d1c-b4de-ec6300395072 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:24Z" message: Started container switch-graph-25c51 metadata: creationTimestamp: "2026-07-01T20:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:24Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ec8ce5e766 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: bccf48d3-af81-4627-b068-e1b72bc647f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:56:32Z" message: Stopping container switch-graph-25c51 metadata: creationTimestamp: "2026-07-01T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:32Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ee4688fa05 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 671520f2-1174-476c-899e-b12c52585f8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-25c51} kind: Pod name: switch-graph-25c51-56c67f6cb4-wlzqd namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ad8dff79-1e91-4a85-8400-9d86376cb3d5 kind: Event lastTimestamp: "2026-07-01T20:57:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:01Z" name: switch-graph-25c51-56c67f6cb4-wlzqd.18be45ef37166733 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 90e6a807-3972-425a-af4b-6dcb3075eb47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-25c51-56c67f6cb4 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: f8dfe010-0889-4cac-83fa-5d38f8d3b8d6 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: 'Created pod: switch-graph-25c51-56c67f6cb4-wlzqd' metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51-56c67f6cb4.18be45ebead5b72e namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 7d8f4182-40b4-4ac1-8a55-be752a80b019 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-25c51 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: af78d127-3acb-4f16-9b7a-75f390d3a71d kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: Scaled up replica set switch-graph-25c51-56c67f6cb4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51.18be45ebe98eacce namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: fba4c6f2-8ae9-4fd1-9f60-e81f43c23a4d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-25c51 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 65e36188-0708-4371-a42e-c5cb3f969fad kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: InferenceGraph [switch-graph-25c51] is Ready metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:56:22Z" name: switch-graph-25c51.18be45ebeef0f583 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: f81f644a-ceb2-4018-8664-901ccf99e843 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:18:02.365349Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ce993-845959cd86-5gstt to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86-5gstt.18be471aabfaccad namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: 62eb6d9e-5c4b-4c14-b1bc-8c6189ef3af7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86-5gstt.18be471ac5bc28a2 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 7cfd858c-600c-4ddd-ae6f-2dcfea8d0a0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce993} kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86-5gstt.18be471ac70bfcca namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 9a2af035-cf75-412c-aafd-c60d1a7705ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce993} kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: 'Created container: switch-graph-ce993' metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86-5gstt.18be471ac9cc7448 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: efa2c7d0-0ae4-42d3-91c4-b34d47e15e53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce993} kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: Started container switch-graph-ce993 metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86-5gstt.18be471aca40857f namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 73a7ef18-4750-4b54-9402-bfa63688b8ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce993} kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:34:21Z" message: Stopping container switch-graph-ce993 metadata: creationTimestamp: "2026-07-01T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:21Z" name: switch-graph-ce993-845959cd86-5gstt.18be47fea5d5e688 namespace: kserve-ci-e2e-test resourceVersion: "33778" uid: 53726306-b1a4-4c21-a479-8a7e07cd90e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:34:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce993} kind: Pod name: switch-graph-ce993-845959cd86-5gstt namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f754b68d-c342-4b58-a248-c68fbc970586 kind: Event lastTimestamp: "2026-07-01T21:34:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:34:44Z" name: switch-graph-ce993-845959cd86-5gstt.18be47ff5cb31bcc namespace: kserve-ci-e2e-test resourceVersion: "34019" uid: 5ae3da10-de60-42ae-8a0b-6e177ef57cc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ce993-845959cd86 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: d208ba28-4bec-4cb5-bcb4-8a2ac95fcd80 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: 'Created pod: switch-graph-ce993-845959cd86-5gstt' metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993-845959cd86.18be471aab617434 namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 864fdd4e-5f2d-40bb-bacd-bf437db7d2be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: f1b6e31c-7ebd-4191-9338-00fc67319e47 kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: Scaled up replica set switch-graph-ce993-845959cd86 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993.18be471aaa41cf96 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: b5d3e09c-60f0-4ee0-bf0a-6eb55ce14248 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: b07dea9f-e8c1-4d24-9508-e7dd5936e00d kind: Event lastTimestamp: "2026-07-01T21:18:02Z" message: InferenceGraph [switch-graph-ce993] is Ready metadata: creationTimestamp: "2026-07-01T21:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:18:02Z" name: switch-graph-ce993.18be471ab0f7aa64 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 5005afe2-60e3-4c17-b0b9-76903f5a8eae reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: ecbda6f6-bf86-4297-be83-5d9ae6bf946a kind: Event lastTimestamp: "2026-07-01T21:18:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:32Z" name: switch-graph-ce993.18be471e28cefc01 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: a910af98-656f-458d-99b3-2f2dd2905493 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce993 namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: ecbda6f6-bf86-4297-be83-5d9ae6bf946a kind: Event lastTimestamp: "2026-07-01T21:18:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:18:32Z" name: switch-graph-ce993.18be471e293fc36b namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 2038580f-8a97-4cb8-9f46-2643ce1d13e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:57:42.401750Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f347e-6f4bb9d8fd-f5wl4 to ip-10-0-130-150.ec2.internal metadata: creationTimestamp: "2026-07-01T20:57:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:57:42Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45fea07ba667 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 8dcfa1f8-f1e6-4d81-83a4-f70daf2b86e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6dfd87768c-9qvnd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T20:57:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-f347e-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T20:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:42Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45fea93fef06 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 1043ad99-a370-471d-b01c-abe246e107c2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T20:57:43Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:57:43Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45fedd9cd771 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 95a7acca-9f2b-4c37-95b6-a01257779b0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f347e} kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T20:57:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:43Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45feded850ae namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: efa13668-77c8-4da5-95b9-4c20c78ba140 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f347e} kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T20:57:43Z" message: 'Created container: switch-graph-f347e' metadata: creationTimestamp: "2026-07-01T20:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:43Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45fee25553ca namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 9db4b67d-1197-4217-be5b-71c79d35178f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f347e} kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T20:57:43Z" message: Started container switch-graph-f347e metadata: creationTimestamp: "2026-07-01T20:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:43Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be45fee2e36835 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 8f47b505-ab32-4206-812f-57de66698ac9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f347e} kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: Stopping container switch-graph-f347e metadata: creationTimestamp: "2026-07-01T21:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be4671d190db0b namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 82969b9d-f80c-45fc-a011-f0e564f62aa3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f347e} kind: Pod name: switch-graph-f347e-6f4bb9d8fd-f5wl4 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 23cc18e6-9ac8-46b2-9b5e-c20702bbd904 kind: Event lastTimestamp: "2026-07-01T21:06:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:24Z" name: switch-graph-f347e-6f4bb9d8fd-f5wl4.18be46725ffeff51 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 48393e8c-7f6b-46ac-979a-68f037b45cfd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-150.ec2.internal source: component: kubelet host: ip-10-0-130-150.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f347e-6f4bb9d8fd namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: eaa44bb5-2760-4fe7-acf1-66db562ea58d kind: Event lastTimestamp: "2026-07-01T20:57:42Z" message: 'Created pod: switch-graph-f347e-6f4bb9d8fd-f5wl4' metadata: creationTimestamp: "2026-07-01T20:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:42Z" name: switch-graph-f347e-6f4bb9d8fd.18be45fe9fe5a396 namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: b57926d9-02fd-4b1e-918e-50e149d19446 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f347e namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: ff4aa84a-6a72-49a3-9361-58761f09d583 kind: Event lastTimestamp: "2026-07-01T20:57:42Z" message: Scaled up replica set switch-graph-f347e-6f4bb9d8fd from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:42Z" name: switch-graph-f347e.18be45fe9ec307d1 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: d3becb7b-fbc4-4a53-9283-7512a3e0096a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f347e namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: b78fd421-38e9-4eca-94a3-6a07ce37fb83 kind: Event lastTimestamp: "2026-07-01T20:57:42Z" message: InferenceGraph [switch-graph-f347e] is Ready metadata: creationTimestamp: "2026-07-01T20:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:42Z" name: switch-graph-f347e.18be45fea439403c namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 4d48c81e-58e5-4b4f-bccb-596ab5703386 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:57:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f347e namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: c24d08bc-152d-4b9b-863d-cd885b7083ff kind: Event lastTimestamp: "2026-07-01T20:58:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:12Z" name: switch-graph-f347e.18be46021d40b5b9 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: 4d88b660-1af1-41c5-9374-0d6d22955cb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:57:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f347e namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: c24d08bc-152d-4b9b-863d-cd885b7083ff kind: Event lastTimestamp: "2026-07-01T20:58:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:12Z" name: switch-graph-f347e.18be46021d959478 namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: 8ffbe901-c7a0-4b49-a4a7-72f92b2ede92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34047"