--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:41:57.725062Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-0ca59-8bcb68dfb-v4b49 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:41:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:41:57Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9cb828e4aa54 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 5d264d67-e064-422b-a965-84a34b3173c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:41:58Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:41:58Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9cb841f0f7b7 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: e8599182-0818-4799-b4d0-6e8be65a39e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0ca59} kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:41:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:58Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9cb84338039f namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: a0efd1fb-f981-4d01-9080-0f9d7a0c025d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0ca59} kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:41:58Z" message: 'Created container: ensemble-graph-0ca59' metadata: creationTimestamp: "2026-04-28T19:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:58Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9cb8472c708d namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 39f70874-b2d8-4a1c-a7f2-afa246bbdfbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0ca59} kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:41:58Z" message: Started container ensemble-graph-0ca59 metadata: creationTimestamp: "2026-04-28T19:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:58Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9cb847aabc8f namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: a804d611-de3a-4bb1-8a58-686ac5074bd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0ca59} kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Stopping container ensemble-graph-0ca59 metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9d2b5c2d7f11 namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: 872b4049-5e26-43be-be09-35361ef411a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0ca59} kind: Pod name: ensemble-graph-0ca59-8bcb68dfb-v4b49 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7ce3f5e5-d1d4-453c-9956-fb5b0e720ca4 kind: Event lastTimestamp: "2026-04-28T19:50:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:39Z" name: ensemble-graph-0ca59-8bcb68dfb-v4b49.18aa9d2bcdcb8dcb namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: 31780501-3cb1-43a0-b2e2-29ef33cd84ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-0ca59-8bcb68dfb namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 7b7a0856-8e11-478c-bbd1-ae1c035d7b10 kind: Event lastTimestamp: "2026-04-28T19:41:57Z" message: 'Created pod: ensemble-graph-0ca59-8bcb68dfb-v4b49' metadata: creationTimestamp: "2026-04-28T19:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:57Z" name: ensemble-graph-0ca59-8bcb68dfb.18aa9cb8288bacf1 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: d1e3006a-7895-4ab5-a440-11f96ac9be5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 994f3190-91a3-4776-8a35-775d4a061e02 kind: Event lastTimestamp: "2026-04-28T19:41:57Z" message: Scaled up replica set ensemble-graph-0ca59-8bcb68dfb from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:57Z" name: ensemble-graph-0ca59.18aa9cb827de4051 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: e4156ee4-95f3-47a8-9134-1a99602ce7e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: d7516e73-11a0-4b43-879b-23f9e0241bfd kind: Event lastTimestamp: "2026-04-28T19:41:57Z" message: InferenceGraph [ensemble-graph-0ca59] is Ready metadata: creationTimestamp: "2026-04-28T19:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:57Z" name: ensemble-graph-0ca59.18aa9cb82ba21013 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 5b2b5b10-5f98-4cf0-8dd3-71f1412acb22 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 973fa774-6c37-494b-87ac-728a829135c8 kind: Event lastTimestamp: "2026-04-28T19:42:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:12Z" name: ensemble-graph-0ca59.18aa9cbba68e7682 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 0bcb1e2c-337e-4f71-869f-e8061f7e84fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 973fa774-6c37-494b-87ac-728a829135c8 kind: Event lastTimestamp: "2026-04-28T19:42:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:12Z" name: ensemble-graph-0ca59.18aa9cbba6ff3db3 namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: 251494d3-3095-4228-be71-ab300e9922fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 973fa774-6c37-494b-87ac-728a829135c8 kind: Event lastTimestamp: "2026-04-28T19:42:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:27Z" name: ensemble-graph-0ca59.18aa9cbf25a6a739 namespace: kserve-ci-e2e-test resourceVersion: "21702" uid: 29c9ab06-1bad-45fc-82aa-515edbfc8b51 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 973fa774-6c37-494b-87ac-728a829135c8 kind: Event lastTimestamp: "2026-04-28T19:42:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:27Z" name: ensemble-graph-0ca59.18aa9cbf260e20de namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 02b94bce-ae20-437d-9e16-69aa1f70ac2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:40:47.439821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-2a210-76dff99855-v7s6j to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:40:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:40:47Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca7cb90f5b5 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: c8f32fc5-637e-4c8f-a448-73d49ff90c7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-2a210-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:47Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca7d3658998 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 6c766e3a-fcd6-4e58-959d-828e17a723a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:48Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:48Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:40:48Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca808dc9288 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: d4de461c-9cb6-41d2-89b9-5f9852764df0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2a210} kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:48Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca80a061cde namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 0713ba57-e219-434d-a31a-92c0a4d1d4c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2a210} kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:48Z" message: 'Created container: ensemble-graph-2a210' metadata: creationTimestamp: "2026-04-28T19:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:48Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca80d176a69 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 8c5cbedb-dbe2-491c-b3b7-1c74e7533a52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2a210} kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:48Z" message: Started container ensemble-graph-2a210 metadata: creationTimestamp: "2026-04-28T19:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:48Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9ca80d8bf881 namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 64e188b4-cc17-43a4-bf8c-c6800dc7f258 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2a210} kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Stopping container ensemble-graph-2a210 metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:57Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9caa23f94e1e namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 43bbde75-73e9-4587-b915-dc4157167a85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2a210} kind: Pod name: ensemble-graph-2a210-76dff99855-v7s6j namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6f6b7d47-ecfc-45df-a4ea-c6a83efedf61 kind: Event lastTimestamp: "2026-04-28T19:41:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:25Z" name: ensemble-graph-2a210-76dff99855-v7s6j.18aa9caac19f5743 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 62ecc345-faf7-4906-8249-04a4554faf08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-2a210-76dff99855 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 07c4ef0c-ef59-48d8-b61f-594ebe14ae83 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'Created pod: ensemble-graph-2a210-76dff99855-v7s6j' metadata: creationTimestamp: "2026-04-28T19:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: ensemble-graph-2a210-76dff99855.18aa9ca7cb2c7504 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 55492097-c821-4d68-aa53-b9909fbf164a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-2a210 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 0c001802-28f6-4491-9545-feb12f717909 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: Scaled up replica set ensemble-graph-2a210-76dff99855 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: ensemble-graph-2a210.18aa9ca7ca93ecbd namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 30dd8ded-b8d1-4af6-b72f-9270ce58e92d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2a210 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 4f13f20d-756a-4aa0-9cfa-889ee44406b8 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: InferenceGraph [ensemble-graph-2a210] is Ready metadata: creationTimestamp: "2026-04-28T19:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:47Z" name: ensemble-graph-2a210.18aa9ca7cea43734 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: f07f9900-6102-4e02-ab03-5232b7a4142c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:30:32.691794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0600b-predictor-cf96ff6c-ccj24 to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18a9ba76d0 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 685ac0a3-cd3e-4c1c-965f-b8538ecaf266 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18c3706446 namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 28588204-51bc-4fb9-840b-ea7539754f53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18c4c0e5fc namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: cb4da86d-68e0-4c59-8139-5dc9fbef19db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18c7b5d95b namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 7b842160-61f2-4252-b165-cbda6255fd32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18c82b9416 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: dd3482fb-313b-4c58-8fcd-e6c68241edb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18c83a5a21 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 16fac826-242a-4420-8bb3-87c8ee958ef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18cb375afd namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 92aaa7c1-3323-499a-ba00-e445890e9390 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c18cbabed4c namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 6441f7cd-808e-4cdb-99d2-9c5a0c67ac06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:39:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:50Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c191393f545 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 003ee9a6-5a7e-4549-95d1-c46d8f599efb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c99d0de7fa3 namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: bb913852-83d8-496b-a643-a82dec6b3ab8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c99d0df2440 namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 294831eb-0436-42d2-bb21-f07121020dc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8a144135-6c0c-4cde-9faa-c8676d399381 kind: Event lastTimestamp: "2026-04-28T19:39:50Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:50Z" name: error-404-isvc-0600b-predictor-cf96ff6c-ccj24.18aa9c9a87e82c04 namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: a240e09f-a407-410c-837e-dd0e5dfb29cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0600b-predictor-cf96ff6c namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: c1f2bb0b-7aad-4874-bb19-3b38ff3a9c99 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'Created pod: error-404-isvc-0600b-predictor-cf96ff6c-ccj24' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-0600b-predictor-cf96ff6c.18aa9c18a92ccbdc namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: fda42746-e311-4451-8676-fa1ef4a254a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: c889fc89-ec17-48e1-856b-c837432c1031 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Scaled up replica set error-404-isvc-0600b-predictor-cf96ff6c from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-0600b-predictor.18aa9c18a776a31b namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: ae1ae4fe-405c-4133-b01e-e73f97d7d7e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 801380c3-58f3-4411-89b5-f2b1037ce317 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: error-404-isvc-0600b-predictor.18aa9c1c2625d40b namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: d147e898-1292-4b71-81bd-b8a45eeee1a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 801380c3-58f3-4411-89b5-f2b1037ce317 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: error-404-isvc-0600b-predictor.18aa9c1c268cc2e6 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: cf754472-8387-489c-9e1a-4cf9e9e1a199 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 801380c3-58f3-4411-89b5-f2b1037ce317 kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:32Z" name: error-404-isvc-0600b-predictor.18aa9c23240eb392 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: e29ecd16-2565-40f9-bd3b-618640cc9474 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 801380c3-58f3-4411-89b5-f2b1037ce317 kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:32Z" name: error-404-isvc-0600b-predictor.18aa9c2324740978 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 820df3fb-4367-477f-9e26-d9b5934af29d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 703439a1-eb40-4199-b0ac-913e73c5531e kind: Event lastTimestamp: "2026-04-28T19:31:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:20Z" name: error-404-isvc-0600b.18aa9c18a77d99a9 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: f7b34bf0-39a5-4d4f-8be0-7f9783e89abc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 703439a1-eb40-4199-b0ac-913e73c5531e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-0600b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0600b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-0600b.18aa9c18a7ff7234 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: e08fe57f-1fa3-4878-8955-0ce6e74dd7ce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 703439a1-eb40-4199-b0ac-913e73c5531e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0600b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-0600b.18aa9c18a7ffcb2c namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: b8ebba10-15a1-484c-ba06-343aea646f7c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 703439a1-eb40-4199-b0ac-913e73c5531e kind: Event lastTimestamp: "2026-04-28T19:31:20Z" message: InferenceService [error-404-isvc-0600b] is Ready metadata: creationTimestamp: "2026-04-28T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:20Z" name: error-404-isvc-0600b.18aa9c23ce544614 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 19c8e166-53be-4010-a108-ba8b65f62244 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:40:57.764882Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0ca59-predictor-546755f487-skcxd to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caa32fce7aa namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 8a57c7ca-1d15-430f-9fd5-6a721dce16e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0ca59-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caa3d2d7ad6 namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 85e04900-3ff0-46aa-a133-893605fa024c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaa5b3f31e namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: dbafdf18-9bb2-416e-a006-95f50ce66ed3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaa6ef43cb namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 7772d5ed-7090-4141-9170-b29c0b0315e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaa9f70ae4 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 36fa607e-92fc-409d-813d-c146229031a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaaa6b6680 namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: 494b2ea0-4278-4721-b669-7b9914670e4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaaa774177 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 62c481dd-16ee-40f0-b460-17ade0e3f981 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaadd237b5 namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: dc3d53c0-2d09-44bd-8e24-2db6c0562449 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:40:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:59Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9caaae477596 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 3abae0ec-1f12-4fe7-8a6c-113363f6b79d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:41:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9cab18f2a999 namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 1ea08261-90d1-469e-9e69-ace461b50670 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9d2b66c6ce86 namespace: kserve-ci-e2e-test resourceVersion: "25287" uid: d3ae2e3f-ac75-4d9d-86ae-29e340688db3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ca59-predictor-546755f487-skcxd namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: f95b93a4-6369-4e6c-a06a-60d166c6ce2a kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-0ca59-predictor-546755f487-skcxd.18aa9d2b66c76dfb namespace: kserve-ci-e2e-test resourceVersion: "25291" uid: 1e20ce4d-9fd4-4308-a050-972cc463e449 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0ca59-predictor-546755f487 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 3ba1d255-f8f3-4f18-8d5d-29a51a14c52f kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'Created pod: error-404-isvc-0ca59-predictor-546755f487-skcxd' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-0ca59-predictor-546755f487.18aa9caa32942ece namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: dbc00c07-ba43-4849-8277-ce7694e65201 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 2230033d-bb88-4220-9706-073c9268a9c4 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Scaled up replica set error-404-isvc-0ca59-predictor-546755f487 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-0ca59-predictor.18aa9caa30d6328d namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: de364743-37d6-4dbd-9a0c-f33c46f7d87d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 629ef7a9-b043-44ad-a141-69824ebe3618 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: error-404-isvc-0ca59-predictor.18aa9cadb4ef43a8 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 3b0122b3-1222-4c8e-a39e-dbe6072cde96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 629ef7a9-b043-44ad-a141-69824ebe3618 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: error-404-isvc-0ca59-predictor.18aa9cadb5807ff1 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 1fbd78d1-5003-49eb-8c74-818ed9020da3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 629ef7a9-b043-44ad-a141-69824ebe3618 kind: Event lastTimestamp: "2026-04-28T19:42:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:12Z" name: error-404-isvc-0ca59-predictor.18aa9cb4b31cce58 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: ebda2ffa-4eeb-45f2-b8eb-4d4247266218 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 629ef7a9-b043-44ad-a141-69824ebe3618 kind: Event lastTimestamp: "2026-04-28T19:42:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:12Z" name: error-404-isvc-0ca59-predictor.18aa9cb4b38f9ee9 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 1203000f-28c1-4107-816b-fa27674d4bd7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: fd877e7c-0778-409b-87bf-ec34f06131c4 kind: Event lastTimestamp: "2026-04-28T19:41:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:47Z" name: error-404-isvc-0ca59.18aa9caa309f578d namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 0daf2697-2758-4237-b99f-0e3084152626 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: fd877e7c-0778-409b-87bf-ec34f06131c4 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-0ca59": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0ca59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-0ca59.18aa9caa310e5c8c namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 7fb34836-0d35-4e24-8c2f-81e386586d6b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: fd877e7c-0778-409b-87bf-ec34f06131c4 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0ca59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-0ca59.18aa9caa310eac49 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 7c87ed6c-c1a7-4e53-b5e1-a6af748feda7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: fd877e7c-0778-409b-87bf-ec34f06131c4 kind: Event lastTimestamp: "2026-04-28T19:41:47Z" message: InferenceService [error-404-isvc-0ca59] is Ready metadata: creationTimestamp: "2026-04-28T19:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:47Z" name: error-404-isvc-0ca59.18aa9cb5d3c93d96 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 019d85fa-4cea-4837-9904-bbcb1630b361 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:40:27.356298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca31e7eb98a namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: e1e43f8b-5c95-4348-8586-b914e59ed64c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca3382993e9 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 79bb7aec-509e-41c3-b594-6eb57203ea21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca33980107c namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 8cdcc254-14be-4b09-9f77-0e99f974d810 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca33c59cb1b namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: 1c23939a-689e-4178-bb26-40d175fc4410 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca33ccc2dcb namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: d44c8ab4-6e19-48a6-ae8f-76cdd9ade4f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca33cd8bf92 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: 61251c21-e3e7-408a-a824-e3f76c300f09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca340b8a82f namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 12d179e8-0cda-4339-adfb-1270a2890062 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca3416739b8 namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 35c43f07-7b12-40ec-b8e2-5f1068118f07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:41:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:05Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9ca39e97cc43 namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 0e27f475-041a-4a68-aed9-9ebc1bfd48bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9cb3753dd072 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 647a8c19-3db0-4019-915b-e936c2b50f00 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9cb3753e4ffc namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 253aec40-5b3e-4ac4-afd0-214367538e26 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4c09e77c-dde3-4787-8002-d63f394b155c kind: Event lastTimestamp: "2026-04-28T19:41:40Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:40Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs.18aa9cb426c64025 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 356edd65-c9d4-4214-b648-d3cdf00abc57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-26ebe-predictor-7d49cd9f75 namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 3c03ed84-ff04-4311-8933-811f9405812f kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Created pod: error-404-isvc-26ebe-predictor-7d49cd9f75-qjrrs' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor-7d49cd9f75.18aa9ca31e1bec99 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: f807e1ed-0c7d-49e4-81df-c87d94852937 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: b5032283-0227-4119-b076-acd3c231c99c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Scaled up replica set error-404-isvc-26ebe-predictor-7d49cd9f75 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe-predictor.18aa9ca31c8140f9 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: e809fb20-de72-46b9-af75-84afec3c97c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 92bd7413-ed82-4284-bb79-b2a0f95bd56c kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: error-404-isvc-26ebe-predictor.18aa9ca69af19a62 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: a398af30-e3f1-451d-bf5a-59283c4e3be9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 92bd7413-ed82-4284-bb79-b2a0f95bd56c kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: error-404-isvc-26ebe-predictor.18aa9ca69b540c31 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: f63bab55-4dc0-4759-8c66-62ed5ca35fff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 92bd7413-ed82-4284-bb79-b2a0f95bd56c kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-26ebe-predictor.18aa9caa19fc4b3b namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 159cf28f-e3ef-4e31-8698-9d6f8b993011 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 92bd7413-ed82-4284-bb79-b2a0f95bd56c kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-26ebe-predictor.18aa9caa1a5f77cd namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 7bc59cb9-bfa6-47ca-8a69-adcc54873ebc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 0cc284f5-af64-460f-9365-219153663d67 kind: Event lastTimestamp: "2026-04-28T19:41:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:15Z" name: error-404-isvc-26ebe.18aa9ca31c6051a8 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: cfa9d17e-204c-48e1-993d-8ee511bafd6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 0cc284f5-af64-460f-9365-219153663d67 kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-26ebe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe.18aa9ca31cb0bb87 namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 01fb5d0b-d9df-4023-adc3-449bd2603c51 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 0cc284f5-af64-460f-9365-219153663d67 kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-26ebe.18aa9ca31cb12584 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: c61ea4de-3cd1-4fb6-8b05-8f3646629291 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 0cc284f5-af64-460f-9365-219153663d67 kind: Event lastTimestamp: "2026-04-28T19:41:15Z" message: InferenceService [error-404-isvc-26ebe] is Ready metadata: creationTimestamp: "2026-04-28T19:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:15Z" name: error-404-isvc-26ebe.18aa9cae59d82dc7 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 91fc66ef-a694-4d28-91bb-367bb125b946 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:39:47.470309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2a210-predictor-7bb85759bc-hqfwt to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99d51ad1dd namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 085b517f-457e-4ca5-a7ad-e13abb64b687 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99ee443390 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: d68701bf-f927-4972-90e4-1e5a393d52b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99ef7fccec namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: bcf7800a-c89e-47be-9b1e-7971eb3a523e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99f2d081ce namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 4390eda0-6e53-4899-be81-93c8ba4f89d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99f3431526 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: 9e578856-b266-4838-a46b-c14cfa88fc30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99f34f383b namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: 7ae1e173-c191-4f2b-8d34-22ee9036823a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:48Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99f780f1b2 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 7ad50dd2-0917-4823-80df-1eca89d3d3f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:48Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c99f7f831b7 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: bcd6f5cb-9ae3-482d-b1b8-88f9576a30fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:40:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:25Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9c9a45b16c64 namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: cacbd1c7-fd8e-4ad0-b2e9-7542fb678ea0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9caa2f3dd5da namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: f04ebce2-864b-4b66-8eb1-156e446eeacd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:57Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9caa2f3e79b2 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 5d870756-7c43-4a16-aceb-022d5b599da4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 88dd0647-a02e-49f5-8d04-2a2022ae3095 kind: Event lastTimestamp: "2026-04-28T19:41:00Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:00Z" name: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt.18aa9caacdc6551c namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: adec9a50-cff2-44a0-8314-c89a6c1d6a62 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2a210-predictor-7bb85759bc namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: 1f903293-527d-4c72-a4f0-111cf9b7a046 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'Created pod: error-404-isvc-2a210-predictor-7bb85759bc-hqfwt' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor-7bb85759bc.18aa9c99d49fbf1a namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: d44eff52-b8cd-4a71-a2bd-330a30139e69 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 431c093b-293c-4eda-8d11-a463f22775bc kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Scaled up replica set error-404-isvc-2a210-predictor-7bb85759bc from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210-predictor.18aa9c99d28ebb1c namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 34a81cf8-47f7-4d0e-85d9-00616ad8bd6f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 2012e027-c908-4c83-a085-f469e7d1a597 kind: Event lastTimestamp: "2026-04-28T19:40:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:17Z" name: error-404-isvc-2a210-predictor.18aa9c9d5108fc27 namespace: kserve-ci-e2e-test resourceVersion: "20036" uid: f4a95fe5-e1c8-4d0e-9ebf-e58133716c38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 2012e027-c908-4c83-a085-f469e7d1a597 kind: Event lastTimestamp: "2026-04-28T19:40:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:17Z" name: error-404-isvc-2a210-predictor.18aa9c9d51772e77 namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: 03767f37-143e-42e3-baaa-8b1b857d1ace reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 2012e027-c908-4c83-a085-f469e7d1a597 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: error-404-isvc-2a210-predictor.18aa9ca44ee34cf9 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 677d49d8-087a-4ead-a811-3fa1f060efb2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 2012e027-c908-4c83-a085-f469e7d1a597 kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: error-404-isvc-2a210-predictor.18aa9ca44f47d318 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: ba530e64-4eae-4112-9c4f-26ea5e589e35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: f1c0abc6-726a-4ea3-924d-e6382f419667 kind: Event lastTimestamp: "2026-04-28T19:40:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:35Z" name: error-404-isvc-2a210.18aa9c99d263a131 namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: efd86273-96d3-4f37-a5c0-7530d016a90c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: f1c0abc6-726a-4ea3-924d-e6382f419667 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-2a210": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2a210": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210.18aa9c99d2d5f8ea namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: cfa7f303-ba24-42a9-9247-2c049e055c07 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: f1c0abc6-726a-4ea3-924d-e6382f419667 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2a210": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:39:47Z" name: error-404-isvc-2a210.18aa9c99d2d64bf0 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 9876e9ff-b994-4f44-8d06-004a769aaadd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: f1c0abc6-726a-4ea3-924d-e6382f419667 kind: Event lastTimestamp: "2026-04-28T19:40:35Z" message: InferenceService [error-404-isvc-2a210] is Ready metadata: creationTimestamp: "2026-04-28T19:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:35Z" name: error-404-isvc-2a210.18aa9ca5008691be namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 3c4cba5c-503c-4759-a2d6-eb29ac289141 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:50:12.772738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2b6c08afa0 namespace: kserve-ci-e2e-test resourceVersion: "25319" uid: 551c120a-3e5c-4c32-8017-83e59074ae3e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6bad9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2b784e46e2 namespace: kserve-ci-e2e-test resourceVersion: "25330" uid: 8aad530a-bc88-4d94-bf53-704ccc789a88 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2ba92d3c03 namespace: kserve-ci-e2e-test resourceVersion: "25351" uid: 0cab9845-ea45-4439-b4ca-5588365b4940 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2baa73cbea namespace: kserve-ci-e2e-test resourceVersion: "25353" uid: ce785a7f-8bb7-43f0-960e-333b7cce7e12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2bad786ab6 namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: d0b2b4b8-2b39-40c3-b5dd-607d95c7f070 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2badeb4486 namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 6abf07b5-3716-40d4-a906-2d23e5f48c42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2badf76d55 namespace: kserve-ci-e2e-test resourceVersion: "25357" uid: f03d8ed0-0273-46a4-91cb-8d15a765cbe0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2bb1223d86 namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 6b52e0ff-9e59-418f-a38b-9706024e3b7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2bb19647c4 namespace: kserve-ci-e2e-test resourceVersion: "25360" uid: fac93674-9bef-4cdb-8773-e960cd215aaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:50:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:50Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d2bd2df8593 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: cf60771c-67f2-4ec5-b449-8cf4a2fd2242 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d3bc69dec61 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: 74670bb5-1d60-48ef-ac3e-c21895b8b745 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d3bc69e71f2 namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 7bea58ec-0a13-4108-b1e1-63e032bdee20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: f62e4801-d6f5-4881-b82f-c90cae86b77c kind: Event lastTimestamp: "2026-04-28T19:51:25Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:25Z" name: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7.18aa9d3c5b04e2fe namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 14451743-195b-4600-9301-ab2041df38ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6bad9-predictor-6c4b946b59 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 34e0f0bd-8d49-4c90-ac3e-65fa1576021b kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'Created pod: error-404-isvc-6bad9-predictor-6c4b946b59-klxc7' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9-predictor-6c4b946b59.18aa9d2b6b97d5fe namespace: kserve-ci-e2e-test resourceVersion: "25316" uid: 24d14b30-6b5c-4893-b300-8fbded486f0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: ea0a35a8-e9a3-46df-b626-3aba686d662e kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Scaled up replica set error-404-isvc-6bad9-predictor-6c4b946b59 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9-predictor.18aa9d2b6a0fc41b namespace: kserve-ci-e2e-test resourceVersion: "25310" uid: 7c727cb4-ccf1-4c53-94ca-79295ce3111a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: 53acc3de-19e9-4566-b1eb-45f211373b8b kind: Event lastTimestamp: "2026-04-28T19:50:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:42Z" name: error-404-isvc-6bad9-predictor.18aa9d2ee88ffa3c namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: a82f45ad-f4b1-47f3-8ad5-d5fa6cb75fef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: 53acc3de-19e9-4566-b1eb-45f211373b8b kind: Event lastTimestamp: "2026-04-28T19:50:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:42Z" name: error-404-isvc-6bad9-predictor.18aa9d2ee8f8b1b1 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: ab457efb-2b0d-4789-87b6-c33b339a7ab3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 53acc3de-19e9-4566-b1eb-45f211373b8b kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: error-404-isvc-6bad9-predictor.18aa9d35e66969cc namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 3a2ff634-30ba-4eb8-b5b7-d20c14344fc1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 53acc3de-19e9-4566-b1eb-45f211373b8b kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: error-404-isvc-6bad9-predictor.18aa9d35e6c95527 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 1e7e5c19-a701-47ab-bae8-5cb9490904b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: e260becd-2208-42ed-bafe-c3ef7412eb55 kind: Event lastTimestamp: "2026-04-28T19:51:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:00Z" name: error-404-isvc-6bad9.18aa9d2b69e6cbff namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: e4878d1d-dfe6-41eb-bb75-ebe031d1b07a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: e260becd-2208-42ed-bafe-c3ef7412eb55 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-6bad9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6bad9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9.18aa9d2b6a368770 namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: a74f71e3-48bb-4a7c-bfa6-158ff6cf0f1f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: e260becd-2208-42ed-bafe-c3ef7412eb55 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6bad9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:12Z" name: error-404-isvc-6bad9.18aa9d2b6a36d165 namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: 132e06a9-7887-438d-afaa-88d711ad0c2d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: e260becd-2208-42ed-bafe-c3ef7412eb55 kind: Event lastTimestamp: "2026-04-28T19:51:00Z" message: InferenceService [error-404-isvc-6bad9] is Ready metadata: creationTimestamp: "2026-04-28T19:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:00Z" name: error-404-isvc-6bad9.18aa9d368dc7e542 namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 26b3bde5-93fd-4a3b-8479-29eae40f40f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:31:12.391967Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c21e80b1823 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: a51b0f9b-9102-42f1-9b4d-e8bb1cf2d794 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6c7b3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c21f38e3bcb namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: b61f15de-1f3d-4913-bcda-f51cfb3790ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c2224fce7d3 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: d7309b01-9992-4051-a59a-93622a1e4908 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c22264162ec namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: d5b635c6-7849-42f0-bbc2-7122e7df8c81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c2229c9c151 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 927fbd0e-a740-4f49-a677-53a277dca5f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c222a53ce30 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 5274b831-a469-44b2-a0e2-493f36c9b01d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c222a637d0f namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: cf0db281-1b94-4b26-aea6-b780e3f8ad3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c222d7a0de9 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 58b39621-32b2-43dd-9a79-3a8fb93152cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c222dec9435 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 4d28b256-4e69-4af2-96ec-d28e8a9c8669 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:31:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:50Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9c226ca58fa3 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: 68f4594e-35c3-46a2-a2c1-d6c95f0982b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9ca31b3ca5aa namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 43a385a4-3eb6-411b-8184-bb7834efba65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9ca31b3e3bb0 namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 86f93b3f-7e1c-4c6f-beca-00a969d4aab6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:40:31Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-28T19:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:31Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9ca41ca1811f namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 81e6825d-ea6c-4e81-b0f3-0cd5246475a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: ad932821-ff2d-48e0-bc3e-f385f539f10c kind: Event lastTimestamp: "2026-04-28T19:40:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: i/o timeout' metadata: creationTimestamp: "2026-04-28T19:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:31Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7.18aa9ca41ce71067 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: b8308a69-678c-4c1b-8e07-4cf4ec220cf8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6c7b3-predictor-5b9464bf9c namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 673062d8-a4c3-46b4-8e07-79974f3c0410 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created pod: error-404-isvc-6c7b3-predictor-5b9464bf9c-l89s7' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3-predictor-5b9464bf9c.18aa9c21e7a8a0e4 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: b4fbb730-057a-4555-8694-a6e6952f4520 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: a8173062-1d53-49dd-b56c-287f61ac9f0a kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Scaled up replica set error-404-isvc-6c7b3-predictor-5b9464bf9c from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3-predictor.18aa9c21e5782992 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 51afac13-e9c4-4f2b-beff-655ae2bb42ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 039e44b9-e6ed-42d2-b093-ab853fe6f065 kind: Event lastTimestamp: "2026-04-28T19:31:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:42Z" name: error-404-isvc-6c7b3-predictor.18aa9c2564483abf namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: d57dc436-c358-42ab-95ac-23816f568fc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 039e44b9-e6ed-42d2-b093-ab853fe6f065 kind: Event lastTimestamp: "2026-04-28T19:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:42Z" name: error-404-isvc-6c7b3-predictor.18aa9c2564a425df namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 21938fba-5f69-4231-ab5b-0ef4e966889d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 039e44b9-e6ed-42d2-b093-ab853fe6f065 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: error-404-isvc-6c7b3-predictor.18aa9c2c624614d9 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 64cc8803-f9c4-422c-ad87-d33f387fcd0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 039e44b9-e6ed-42d2-b093-ab853fe6f065 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: error-404-isvc-6c7b3-predictor.18aa9c2c62a63980 namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 40d6e8d4-a9c5-4353-a5c6-a4f1e482f18a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 98b96830-d615-4672-9c72-f65f3134d6c0 kind: Event lastTimestamp: "2026-04-28T19:32:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:32:00Z" name: error-404-isvc-6c7b3.18aa9c21e5b33579 namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: c1b9390b-7370-4cee-b3b5-db7cc771f4ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 98b96830-d615-4672-9c72-f65f3134d6c0 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-6c7b3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c7b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3.18aa9c21e679ce02 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 201f45f7-5042-4c69-b1c2-94af0f8d1bed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 98b96830-d615-4672-9c72-f65f3134d6c0 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c7b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:12Z" name: error-404-isvc-6c7b3.18aa9c21e67a287f namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 3ffca02c-0ab3-4c8d-81d0-1a27066e6bef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: 98b96830-d615-4672-9c72-f65f3134d6c0 kind: Event lastTimestamp: "2026-04-28T19:32:00Z" message: InferenceService [error-404-isvc-6c7b3] is Ready metadata: creationTimestamp: "2026-04-28T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:32:00Z" name: error-404-isvc-6c7b3.18aa9c2d2761be66 namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: 20fe26af-c43d-4c63-af4c-54537cf663d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:41:37.601423Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7809f-predictor-858cb9b479-84qdj to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb3796e820c namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 3a38e4a1-ff29-43cf-b853-239347fa9775 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb392bc9751 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: cad8150e-d159-4648-94ee-5af65fdddd97 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb3941e7e3b namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 30cbb3bd-c518-4c8f-b72e-4baf63e648e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb39730b00c namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 7e785ca5-9569-434e-9036-4b989ee3adb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb397a419da namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 4307ede9-4a04-4454-81b5-c71acf58ec2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb397b0b7e6 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: f92306e3-dc81-48a3-93aa-777b4f5e5568 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb39a66e5c1 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: a6f07946-b3c2-4494-a775-544934bda682 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb39ada007a namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 71c522cb-fd45-4772-b0e0-9e04a066d3b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:50:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:55Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9cb3f9b00cb8 namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: 11e5a61b-ce0f-4258-bf5b-b0e5b0bf6385 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9d34aa648b0b namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: d75031a6-01c4-4768-bda1-b1d7c5b9b5ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9d34aa6519bd namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: 1c1b1041-4568-441c-8404-aadaf16a4ec2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7809f-predictor-858cb9b479-84qdj namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: bf672cba-15ab-4de9-a2ce-8605fce8f1ec kind: Event lastTimestamp: "2026-04-28T19:50:55Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:55Z" name: error-404-isvc-7809f-predictor-858cb9b479-84qdj.18aa9d356e1062a0 namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: 47421097-c5ae-4d51-9a5b-741565ea2890 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7809f-predictor-858cb9b479 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 145bff58-cfe6-4437-887d-5f9aa0c350c6 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'Created pod: error-404-isvc-7809f-predictor-858cb9b479-84qdj' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-7809f-predictor-858cb9b479.18aa9cb378a63dab namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 12b9f0ae-c741-4c69-b849-ba921f4ebce0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: eff7e8dc-1d15-4216-aa52-b46d96b93ddf kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Scaled up replica set error-404-isvc-7809f-predictor-858cb9b479 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-7809f-predictor.18aa9cb376d81096 namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: 54f8aba1-cecb-4eb8-99da-06c325110457 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: a8fe5c10-b58a-4ff1-b56b-badfb7956388 kind: Event lastTimestamp: "2026-04-28T19:41:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:52Z" name: error-404-isvc-7809f-predictor.18aa9cb6f56a0e3a namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: c7613b9b-63b7-4dae-bd85-46a1ab00a26f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: a8fe5c10-b58a-4ff1-b56b-badfb7956388 kind: Event lastTimestamp: "2026-04-28T19:41:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:52Z" name: error-404-isvc-7809f-predictor.18aa9cb6f5f6788c namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: ce310db1-fc69-4739-a933-2531a564cf0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: a8fe5c10-b58a-4ff1-b56b-badfb7956388 kind: Event lastTimestamp: "2026-04-28T19:42:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:52Z" name: error-404-isvc-7809f-predictor.18aa9cba74a8f7a7 namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 6f43d5df-d4e4-4fc7-ade7-9ba9768c9c97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: a8fe5c10-b58a-4ff1-b56b-badfb7956388 kind: Event lastTimestamp: "2026-04-28T19:42:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:52Z" name: error-404-isvc-7809f-predictor.18aa9cba750e48dc namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 72641290-0be5-4de0-be41-1045279b3212 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 3c17ece3-9ef4-4a2e-98c9-5d16e5052887 kind: Event lastTimestamp: "2026-04-28T19:42:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:42:25Z" name: error-404-isvc-7809f.18aa9cb376b78671 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: a0ce3a52-550f-438e-b376-04f976eb96c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 3c17ece3-9ef4-4a2e-98c9-5d16e5052887 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-7809f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7809f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-7809f.18aa9cb3770b5f3e namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 7070d774-f9af-4da8-958e-2b8f84cbf00b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 3c17ece3-9ef4-4a2e-98c9-5d16e5052887 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7809f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:37Z" name: error-404-isvc-7809f.18aa9cb3770bada1 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 2a56d6b3-dde8-485a-900e-62e04fd1b0e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 3c17ece3-9ef4-4a2e-98c9-5d16e5052887 kind: Event lastTimestamp: "2026-04-28T19:42:25Z" message: InferenceService [error-404-isvc-7809f] is Ready metadata: creationTimestamp: "2026-04-28T19:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:42:25Z" name: error-404-isvc-7809f.18aa9cbeb4a96ead namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: a41a7d09-0c11-4c3d-9f68-63b6eff16c5f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:51:23.069165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-93cee-predictor-787774f689-d62cr to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3bca070f41 namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: cb1a7379-c767-45cb-91b5-b20ed00bef58 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-93cee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3bd71a97c1 namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 86439f7f-838f-41f5-b075-49a5f3967d1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26227" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c075893e8 namespace: kserve-ci-e2e-test resourceVersion: "26254" uid: 0592cb7d-fd4c-4746-a7c5-5ed484e72283 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c08a2f8fc namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 53ac7557-5021-444e-8fb5-53659a9a3a25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c0c5e1387 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: b9d691f5-7ea5-4000-ab7b-48322a447282 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c0cd3303c namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: f1c93b4d-5015-44e0-9ae9-7d5f0ff57ef7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c0cdfc211 namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 9776eecf-92d8-4a97-8e8f-f2f583af39d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c10b57227 namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: e56dd0e5-d2d8-46e7-b4fe-1862f0272166 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:51:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:24Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c1130a58f namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 34b72db7-a241-4bab-ad6f-1c7169daeb94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T19:52:01Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:01Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9d3c69fa5208 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: b10defe6-d0f3-4bca-b273-8d0659fe099d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T20:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T20:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:00:37Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9dbcfc3c504b namespace: kserve-ci-e2e-test resourceVersion: "30312" uid: 422c6501-87c9-485b-a023-29c968d6147d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93cee-predictor-787774f689-d62cr namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 7f19c733-98a3-4e90-8c29-4bd9394f8d68 kind: Event lastTimestamp: "2026-04-28T20:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T20:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:00:37Z" name: error-404-isvc-93cee-predictor-787774f689-d62cr.18aa9dbcfc3cffd4 namespace: kserve-ci-e2e-test resourceVersion: "30313" uid: a319774f-1c20-421f-9dde-6a2a28a1caaf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-93cee-predictor-787774f689 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: b2115a43-ab83-4cf7-8331-05117a16d7db kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'Created pod: error-404-isvc-93cee-predictor-787774f689-d62cr' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee-predictor-787774f689.18aa9d3bc9a1e51b namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: c42f4a0d-1ecc-495d-9b4c-29f1a983fc7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 0979720a-c294-4b8a-bd88-d96fc0d6e56f kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Scaled up replica set error-404-isvc-93cee-predictor-787774f689 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee-predictor.18aa9d3bc818649e namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 55bd0e7c-9c11-406a-bec6-d61b95b6de6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 2d088f7f-3b4b-4769-a073-ce0b8467d19e kind: Event lastTimestamp: "2026-04-28T19:52:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:08Z" name: error-404-isvc-93cee-predictor.18aa9d3f469d50f0 namespace: kserve-ci-e2e-test resourceVersion: "26689" uid: acc85886-9500-4808-b189-c729debcf55e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 2d088f7f-3b4b-4769-a073-ce0b8467d19e kind: Event lastTimestamp: "2026-04-28T19:52:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:08Z" name: error-404-isvc-93cee-predictor.18aa9d3f4706f419 namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: db3eed60-d584-4a9e-96fe-5893161b5aab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 2d088f7f-3b4b-4769-a073-ce0b8467d19e kind: Event lastTimestamp: "2026-04-28T19:52:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:38Z" name: error-404-isvc-93cee-predictor.18aa9d42c5b66aac namespace: kserve-ci-e2e-test resourceVersion: "26973" uid: 5a9b3fd3-22d4-4ba0-ad67-bf75d11fcf86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 2d088f7f-3b4b-4769-a073-ce0b8467d19e kind: Event lastTimestamp: "2026-04-28T19:52:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:38Z" name: error-404-isvc-93cee-predictor.18aa9d42c61d7f4c namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: 171effe5-816f-4583-8a12-8ae18f1efbc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 7cf7f9fe-c301-4a09-8385-2364df066e14 kind: Event lastTimestamp: "2026-04-28T19:52:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:52:11Z" name: error-404-isvc-93cee.18aa9d3bc7f77cd8 namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: 4ee89c00-10d0-47d1-8865-25bd63a6e3e9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 7cf7f9fe-c301-4a09-8385-2364df066e14 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-93cee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-93cee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee.18aa9d3bc89abedd namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: b137a9c9-99ec-45b5-8cda-b9a5bb24f4a7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 7cf7f9fe-c301-4a09-8385-2364df066e14 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-93cee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:23Z" name: error-404-isvc-93cee.18aa9d3bc89b0c6e namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 0a8d6678-d108-4258-83fc-275114cb1b81 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 7cf7f9fe-c301-4a09-8385-2364df066e14 kind: Event lastTimestamp: "2026-04-28T19:52:11Z" message: InferenceService [error-404-isvc-93cee] is Ready metadata: creationTimestamp: "2026-04-28T19:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:52:11Z" name: error-404-isvc-93cee.18aa9d4726e5e598 namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 122091cf-8ba4-4ec9-ab0f-4f2612b854cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:02.406044Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e1324-predictor-69874b497-dbzwh to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:02Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c03a4473cac namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 00ab3059-fdde-447a-bd0a-68e9a3d99cb9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:02Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c03bdd7abc0 namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: 0156490e-ff83-466b-9511-bd8a10e4fc5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:02Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c03bf0ff159 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 6c97ad84-ce90-4761-9cb8-a5643ff8406b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" in 14.464s (14.464s including waiting). Image size: 1334713979 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:17Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c071d34785c namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 43654df5-1ab7-4468-977c-d6ec9160ad42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:17Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07215e03b9 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: e8698a26-fd6d-4d60-8acc-ef88cec6c24d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:17Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07223ee0f0 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 45c131c9-c49b-432e-87c9-2246244cd2bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:17Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07224d8180 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 88e7a140-22a7-44bd-a39f-dbbdb34611cd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.514s (2.514s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:19Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07b82e1de1 namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 2c272985-e98e-4707-aa75-eb3ff6bb633c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:19Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07bb6f2a71 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: af6904e2-157f-4a3b-b611-64de9402b40d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:29:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:19Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07bbe064ee namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 553ea8ce-feb5-4090-b26b-cc76d9889c6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-28T19:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:30:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:36Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c07c79d32ff namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: b5f95de3-55bf-47f1-a392-c356bfc99106 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c18a04f35c3 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 84df7fd7-cf26-43c5-99bd-1b78b119d8f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c18a04fbf71 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: de6146d3-33ed-4ee3-901e-018b60a6db45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1324-predictor-69874b497-dbzwh namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 56a93064-6013-47ba-8543-0becf0fb0fb4 kind: Event lastTimestamp: "2026-04-28T19:30:36Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:36Z" name: error-404-isvc-e1324-predictor-69874b497-dbzwh.18aa9c197a3278c0 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: e4bf62d2-44f7-45c1-ad61-e58513466d0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e1324-predictor-69874b497 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: f9b872b9-2f83-42e6-96f4-bf2d74f3ab0f kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: 'Created pod: error-404-isvc-e1324-predictor-69874b497-dbzwh' metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: error-404-isvc-e1324-predictor-69874b497.18aa9c03a3ba2fe3 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: aac6e89a-0a39-4901-a986-0bb652663e65 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: dc0f4aad-89b5-48be-83d7-a052f65bed3f kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Scaled up replica set error-404-isvc-e1324-predictor-69874b497 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: error-404-isvc-e1324-predictor.18aa9c03a12ce1cc namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: 5ef2840e-2d90-44a7-99be-1d92e348242f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 6eb2f693-8468-4f02-b1ba-0e43a05b71f5 kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: error-404-isvc-e1324-predictor.18aa9c071fcbf4ef namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 677c8eaf-e918-4846-8973-d2850dcded1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 6eb2f693-8468-4f02-b1ba-0e43a05b71f5 kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: error-404-isvc-e1324-predictor.18aa9c07209ce87a namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 3c944e24-b892-43b7-b8f0-6146ceaca769 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 6eb2f693-8468-4f02-b1ba-0e43a05b71f5 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-e1324-predictor.18aa9c1519e459e2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 05cc5557-0a50-4486-8531-a8e9dbf264ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 6eb2f693-8468-4f02-b1ba-0e43a05b71f5 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: error-404-isvc-e1324-predictor.18aa9c151a42cda5 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 23a63998-858d-4139-95e9-0f6012fb0133 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 422ddebd-f6c7-4f20-8ff7-c0451a9c82ac kind: Event lastTimestamp: "2026-04-28T19:30:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:16Z" name: error-404-isvc-e1324.18aa9c03a1046650 namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 007f4752-93de-4b7e-8d3d-0b731db2ec74 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 422ddebd-f6c7-4f20-8ff7-c0451a9c82ac kind: Event lastTimestamp: "2026-04-28T19:30:16Z" message: InferenceService [error-404-isvc-e1324] is Ready metadata: creationTimestamp: "2026-04-28T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:16Z" name: error-404-isvc-e1324.18aa9c14d7b8c736 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 84f0e8ad-ff54-4edb-acd6-e9180abc9a01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:50:52.521544Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25758" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34ad3f5630 namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: e08776b2-1ebf-4ed4-a895-879857ba9921 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34c65c9660 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 49936107-0ff5-452d-869b-8e465b4dafe3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34c7ac2a4f namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: cd15289f-6a5f-484d-b66c-f924602cfb5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:53Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34cb0db0e9 namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: e8853240-5342-4381-a59b-3d9a278f6e46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:53Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34cb82dc00 namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 2b30cc51-ea3c-47cf-a345-25c32b727ea3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:53Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34cb8fd038 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: e8954ef5-9269-4580-9e18-46fc42a90af5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:53Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34d0262501 namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 1a074b73-fc1d-49fc-b6b0-fddf3ac0c306 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:50:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:53Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d34d09bf3a0 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: a9cac59d-2016-4ed7-95a6-8e6006e349b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T19:51:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:30Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9d352ba957c6 namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 3d3b4445-e048-460f-9f82-e6bd757f3fe6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T20:08:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T20:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:11Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9e26b1703fc3 namespace: kserve-ci-e2e-test resourceVersion: "33455" uid: cc42f4c3-60b4-492d-9f4b-62d5521e8caf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: 09058041-a3b9-4649-94c5-2f2bbc9e451c kind: Event lastTimestamp: "2026-04-28T20:08:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T20:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:11Z" name: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7.18aa9e26b1711b33 namespace: kserve-ci-e2e-test resourceVersion: "33456" uid: 50c7c244-fc90-4cba-8fa9-e74ea27be493 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e5e5a-predictor-55d7d95565 namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: 41829800-1ea7-419c-aef6-52730055d13a kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Created pod: error-404-isvc-e5e5a-predictor-55d7d95565-l7vg7' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a-predictor-55d7d95565.18aa9d34acda4d48 namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: 500b4803-f1a2-4924-8d74-3877a0625a6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 1627ebf0-3a1c-46c9-89ec-979dc91dd19f kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Scaled up replica set error-404-isvc-e5e5a-predictor-55d7d95565 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a-predictor.18aa9d34ab106fd1 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 3e1fdaed-ca96-452f-aab6-a059771f8fa9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 32bbafcf-3646-434a-8daa-1074484f8a4f kind: Event lastTimestamp: "2026-04-28T19:51:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:07Z" name: error-404-isvc-e5e5a-predictor.18aa9d382993be3f namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: feeafe5f-0a2a-4bb1-849e-5132c15b253b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 32bbafcf-3646-434a-8daa-1074484f8a4f kind: Event lastTimestamp: "2026-04-28T19:51:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:07Z" name: error-404-isvc-e5e5a-predictor.18aa9d3829ef61a6 namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: d28710fc-9092-45da-b0f5-7816d424e667 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 32bbafcf-3646-434a-8daa-1074484f8a4f kind: Event lastTimestamp: "2026-04-28T19:52:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:22Z" name: error-404-isvc-e5e5a-predictor.18aa9d3ba8a1c83c namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: 37e16c62-4ba2-4806-8072-70b1abc2268b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 32bbafcf-3646-434a-8daa-1074484f8a4f kind: Event lastTimestamp: "2026-04-28T19:52:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:22Z" name: error-404-isvc-e5e5a-predictor.18aa9d3ba8e9900f namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: d69bb3a2-c15f-46b2-a927-8b5fb2438375 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 44b5c7c0-91fa-4f5c-9b82-9922e589be47 kind: Event lastTimestamp: "2026-04-28T19:51:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:40Z" name: error-404-isvc-e5e5a.18aa9d34aaec8055 namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 57770c67-81b2-4e47-a38f-581996f165db reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 44b5c7c0-91fa-4f5c-9b82-9922e589be47 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-e5e5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e5e5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a.18aa9d34ab3ae911 namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 85d9cd07-d485-40d1-ad67-c298ba32df02 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 44b5c7c0-91fa-4f5c-9b82-9922e589be47 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e5e5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:52Z" name: error-404-isvc-e5e5a.18aa9d34ab3b44db namespace: kserve-ci-e2e-test resourceVersion: "25755" uid: 02a6b04e-9774-406a-ad39-e9010fbd5f0d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 44b5c7c0-91fa-4f5c-9b82-9922e589be47 kind: Event lastTimestamp: "2026-04-28T19:51:40Z" message: InferenceService [error-404-isvc-e5e5a] is Ready metadata: creationTimestamp: "2026-04-28T19:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:40Z" name: error-404-isvc-e5e5a.18aa9d3fe6a2d62f namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: df028007-d6ae-43ad-93d8-ec8b17b62de6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:02.380397Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c03a2c0010d namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: e5946826-783e-423a-8fca-9df76289ac63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c03bcacaf79 namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: bc674280-3384-4456-b249-4e467e1ba30a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c03bdfa1f12 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: f5d5789f-423b-4d2d-a5cc-832fa32d24d8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" in 12.893s (12.893s including waiting). Image size: 301485528 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c06be7c843f namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 6317061a-e580-4e29-a1e4-63637a1dd216 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c06c23ee4f4 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 9a24550e-a7ef-4039-a286-ae9c2e43f1e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c06c2baad62 namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 6aab5b6a-45d8-49e2-b2a1-29a8ad68c14e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:19Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" metadata: creationTimestamp: "2026-04-28T19:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:19Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c07b3bc8250 namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 6b5ee3cb-823c-41bf-9505-2a39a993310e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" in 7.471s (7.471s including waiting). Image size: 1560922562 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c09710bf106 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: b735b144-6e01-4a3d-b079-601ac26aecf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c0974272123 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 4883a4df-200e-41f6-9335-dc51040fb8df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c0974b7aacb namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: 8d8c8d7a-3e81-4dce-bf75-632ef39d99f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c0974c4423a namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 3c089392-bca2-4253-a126-384522139727 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c09782a2f1b namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 00a1a5f3-8e3c-40cf-a935-dbd58431ba4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c0978a73be4 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 2312d366-0ab3-442f-a884-46c60dab9e11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c099265ca22 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: df501498-8ac0-4b1c-8aec-1beff2708de0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c21e0ec9a04 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: e4d4158f-5561-4026-bc84-caef2183d3b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c21e0ed15eb namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 7f9d324c-2922-4989-aac9-f87ee83d9b57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: ee494290-a65e-414a-9378-97f2c8034eaa kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv.18aa9c2240acdafd namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 0e8d64a9-b6d7-469c-b3ae-b9f70699cae9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7b45b6d56d namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 5116eb53-7100-463b-a1ab-9a2af6f111bf kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7b45b6d56d-gdxfv' metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-1-predictor-7b45b6d56d.18aa9c03a23a425f namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: f4986805-35d7-4c9b-8d46-aaa7e397863d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: a6861f07-184c-4b9a-9e22-27e4ae04d7c8 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7b45b6d56d from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-1-predictor.18aa9c039c2a4cb7 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: f0c9b2cf-5c58-435c-b7ab-238ff392e7ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 7cedcd25-8e2a-4c3e-9f9b-61a58b7fe77e kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: isvc-sklearn-graph-1-predictor.18aa9c071b90981c namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: e6d5dddc-3989-48a3-81b7-35d5c38a40a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 7cedcd25-8e2a-4c3e-9f9b-61a58b7fe77e kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: isvc-sklearn-graph-1-predictor.18aa9c071be2f59e namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: b9aad5f6-a630-4357-bb03-d606158c4042 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 7cedcd25-8e2a-4c3e-9f9b-61a58b7fe77e kind: Event lastTimestamp: "2026-04-28T19:30:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:47Z" name: isvc-sklearn-graph-1-predictor.18aa9c1517fcb8c2 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: c29b723b-d440-417f-83ba-2b6228b18942 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 7cedcd25-8e2a-4c3e-9f9b-61a58b7fe77e kind: Event lastTimestamp: "2026-04-28T19:30:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:47Z" name: isvc-sklearn-graph-1-predictor.18aa9c1518618153 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: d4e5b3ae-02e6-4fd9-9950-e325fa88d29e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: cf6cbe00-7a8f-41dc-999f-45d1ce60e9fa kind: Event lastTimestamp: "2026-04-28T19:30:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:34Z" name: isvc-sklearn-graph-1.18aa9c039c0af08a namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 282ef6c9-a970-43d4-aabb-f09998393ca7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: cf6cbe00-7a8f-41dc-999f-45d1ce60e9fa kind: Event lastTimestamp: "2026-04-28T19:30:34Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-28T19:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:34Z" name: isvc-sklearn-graph-1.18aa9c190ed547df namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 1fc5e03e-08cc-4c2d-87ad-627593e30973 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:03.253095Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-67687994d4-tgjwl to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:03Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c03d6c4475f namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 304df7fe-c233-4cfc-af58-c08b9ee8ce41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:03Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:03Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c03f4fc8a59 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 5d0dfecd-bd33-4cd5-8b61-6a400777989e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:03Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:03Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c03f686829a namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 98c7b2d2-36e6-4056-b6d8-70a61ce71bbb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" in 12.906s (12.906s including waiting). Image size: 301485528 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c06f7d04090 namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 11a5f76e-6af0-460e-a10c-e191c988182a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c06fe1f17a6 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 0417ecc3-659c-4b78-9a9b-0603a4ffc802 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c06ffbc0843 namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 6badde12-797c-4a53-962d-f22d197d77b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" metadata: creationTimestamp: "2026-04-28T19:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:21Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c0803c97586 namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: f4fd2b6b-5c27-4e32-8fe5-0569d8573323 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" in 6.664s (6.664s including waiting). Image size: 1560922562 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c099108e218 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 3e1a06bf-769a-49f4-a609-1568f73ba1a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c0994a83411 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 4ecf4679-00be-45fb-9c69-a7d927a50d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c0995296b04 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e074b0ac-8ba2-4472-8516-a61c7e775772 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c0995494123 namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: cb0c09d8-26ce-4db8-8469-a1191bc6bd20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:28Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c0999df16bf namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 00757aaa-9704-4ac0-9cf5-5e9b24cafb20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:29:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:28Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c099a6b6eac namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 85f217b5-dc78-4f9b-a927-c48bd29d6c65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:31:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:14Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c09a72ef3d9 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 315dfa0a-dfb8-4d1d-9226-478855b3086c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c21de35036d namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: e4be2a4a-7931-4cf8-b6d3-334e02ca4938 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c21de3588b1 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 77d44fc5-0a2f-4584-b477-959b4a45ae44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 6a5b296b-494e-46cf-8143-4b52f1b06ad2 kind: Event lastTimestamp: "2026-04-28T19:31:14Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:14Z" name: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl.18aa9c225571e530 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: d1671785-8e6e-4ad9-b035-eca89669b5ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-67687994d4 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 844d0395-e7c0-405a-bae1-1f26dcb47a17 kind: Event lastTimestamp: "2026-04-28T19:29:03Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-67687994d4-tgjwl' metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:03Z" name: isvc-sklearn-graph-2-predictor-67687994d4.18aa9c03d659e410 namespace: kserve-ci-e2e-test resourceVersion: "14369" uid: c42efc17-6399-4c95-965f-27577d2d9f15 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: e85136e8-470c-474b-ae55-ff22bcf0a767 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-67687994d4 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-2-predictor.18aa9c03c00ee74b namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: ce9ee521-801d-418e-8942-916dcd3c9c09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 1800d347-49d8-48a6-966b-aa668b3c8252 kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: isvc-sklearn-graph-2-predictor.18aa9c073e8cf604 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 862dc5f7-8353-4620-ae48-dec112a4fa00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 1800d347-49d8-48a6-966b-aa668b3c8252 kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:02Z" name: isvc-sklearn-graph-2-predictor.18aa9c073ef36552 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 59b53e3d-7ead-4185-a589-99d4c9614a95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 1800d347-49d8-48a6-966b-aa668b3c8252 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: isvc-sklearn-graph-2-predictor.18aa9c1539f89c30 namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: c9f52c5c-9964-4905-accd-a8836bda8007 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 1800d347-49d8-48a6-966b-aa668b3c8252 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: isvc-sklearn-graph-2-predictor.18aa9c153a5bac5b namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 027988b0-d6c4-4f4f-a2ac-d3d6584865a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 313a3bd8-44f7-43fa-89e3-3f67b7cddd18 kind: Event lastTimestamp: "2026-04-28T19:30:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:44Z" name: isvc-sklearn-graph-2.18aa9c03bfe75796 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 0e64a5a1-d39c-4493-abb3-95953238d3c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 313a3bd8-44f7-43fa-89e3-3f67b7cddd18 kind: Event lastTimestamp: "2026-04-28T19:30:44Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-28T19:30:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:44Z" name: isvc-sklearn-graph-2.18aa9c1b5e44cd9f namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 9eb4c286-c900-45a4-97e9-33413138aa4f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:02.652472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-pcrdj to ip-10-0-138-34.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c03b2f7828d namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: dbf4285a-3b71-436b-84bd-b4ebf0319613 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:03Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c03cc940cd8 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 299dfc73-fa9a-4764-bb57-e79df89f6b5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:03Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" metadata: creationTimestamp: "2026-04-28T19:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c03cdf5d74c namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 13c82ae1-5810-4847-9f27-11e27e4375e5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" in 13.586s (13.586s including waiting). Image size: 301485528 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c06f7c8303f namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: dc5a8f34-8bf0-4de4-a850-a81ea9082087 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c06fe1c79c0 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 1495881b-4264-44a4-9c1c-f3bda4e648e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c06ffbc3cdb namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 83704a82-ace9-45ca-9c15-98bdd9560805 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:21Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-28T19:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:21Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0803e32312 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 34dfbb35-81f7-4067-a8aa-e0bce36c450a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:40Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.679s (19.679s including waiting). Image size: 1306329851 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0c98e6efba namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 59f77856-bac0-4995-bd6f-8c1a63d09eab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0c9ed4eca5 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 05af37df-aead-4ace-a5c8-b9dd7f42a1d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0ca008e58e namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 684a7972-123b-4e75-a5a7-a2da1a5c16fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0ca020cc37 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: cb5a3ad2-8044-4165-9acf-9a9ccee5d9cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0ca4d64f2b namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: d542bb81-200e-47bd-b60c-026666dc88bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:29:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0ca619c0bc namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 8a91462d-d56b-463f-83f5-a2f20aebf07f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-28T19:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:30:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c0cecf6d17d namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 3e29a3f1-a3bb-41e6-8ed1-77fc89810413 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c21e757a9b7 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 67495054-4d99-4fb2-a56e-f2f60080d41e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c21e7582d98 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 32c0225e-ed1c-4557-8df6-6a8c625d2bce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 273c4306-c7c4-4a66-98bd-d68e8649ae68 kind: Event lastTimestamp: "2026-04-28T19:31:13Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:13Z" name: isvc-xgboost-graph-predictor-669d8d6456-pcrdj.18aa9c221d27766a namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ce1524b5-b8ef-431f-a648-f8730740c998 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-34.ec2.internal source: component: kubelet host: ip-10-0-138-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 46175d9f-4ec6-4ffe-8c44-3dd51bd2cabc kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-pcrdj' metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-xgboost-graph-predictor-669d8d6456.18aa9c03b29561a5 namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: a734cd77-ee80-467f-b670-9dd3707bc89d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 29ed39b1-3e34-425c-bbbe-dbbfb2ad50c7 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-xgboost-graph-predictor.18aa9c03a8fcc445 namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 93ff43cf-cac6-4c41-a102-75f473b2de6c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 94a581ab-096c-4a4b-ac4f-586a4f33b510 kind: Event lastTimestamp: "2026-04-28T19:30:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:17Z" name: isvc-xgboost-graph-predictor.18aa9c0727754d61 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 5fa00aa5-72f5-4db7-8026-afddafe28c01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 94a581ab-096c-4a4b-ac4f-586a4f33b510 kind: Event lastTimestamp: "2026-04-28T19:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:17Z" name: isvc-xgboost-graph-predictor.18aa9c0727d8e8ec namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: f06c0d5b-872c-4d1d-96a0-af4c46ec6ab1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 94a581ab-096c-4a4b-ac4f-586a4f33b510 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: isvc-xgboost-graph-predictor.18aa9c18a31854a1 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 7802c764-3089-43be-91bc-e739d58aa043 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 94a581ab-096c-4a4b-ac4f-586a4f33b510 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: isvc-xgboost-graph-predictor.18aa9c18a3bcb87e namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: ac5f3783-7c74-4daa-a7f7-65c37eafdda7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: d1b19fe8-3235-470b-8c37-90eaa288f08c kind: Event lastTimestamp: "2026-04-28T19:30:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:48Z" name: isvc-xgboost-graph.18aa9c03a8da9eb0 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 2b8d2a4c-dd4d-447e-92cc-28ac0a6463cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: d1b19fe8-3235-470b-8c37-90eaa288f08c kind: Event lastTimestamp: "2026-04-28T19:30:48Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-28T19:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:48Z" name: isvc-xgboost-graph.18aa9c1c502f7641 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: f2b868e8-a44c-4aa9-820b-8f78e87e27b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:31:02.031045Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-59fd89b477-szskd to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477-szskd.18aa9c1f7e7beddb namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 3dfd8c6d-341f-4dd7-b3bb-e0a0265f2329 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477-szskd.18aa9c1f984e41b7 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 98ed119b-4609-4b41-a06d-02e65a681459 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477-szskd.18aa9c1f9966ae50 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 1d655f62-ffa2-4b55-888b-eb34ae16bbb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477-szskd.18aa9c1f9c5a04da namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 8bf8620e-66bb-4f1b-be0e-d7e2ee300b61 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477-szskd.18aa9c1f9cd881b1 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 46353e09-8008-49b6-ab45-b1905b54d439 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: model-chainer-59fd89b477-szskd.18aa9c21d73a3372 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 7e1d101f-1739-488e-b6fd-88df45790ad5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59fd89b477-szskd namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5d500d4c-1229-4256-848b-499bd86547a4 kind: Event lastTimestamp: "2026-04-28T19:31:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:39Z" name: model-chainer-59fd89b477-szskd.18aa9c2254b7e0a0 namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: c94c61ca-7e06-4488-86dc-ab8ac1837175 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-59fd89b477 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 10f4e2ec-2a91-476e-b1c6-fbd7ef57ed87 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'Created pod: model-chainer-59fd89b477-szskd' metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer-59fd89b477.18aa9c1f7e078256 namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 6028e8a1-226c-4a00-906a-517fbc906ee8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 863ef241-05dc-4f50-8fcb-9b0779d2fffe kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: Scaled up replica set model-chainer-59fd89b477 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer.18aa9c1f7cf3dd94 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 8ae4991c-618f-4bdb-9bc2-7efbc7279a71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 9b8d2615-3aec-4fa3-8607-8c13faba1fb0 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-28T19:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:02Z" name: model-chainer.18aa9c1f80aabb61 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: aa935ac4-fea6-4bbe-bfa3-01e7482fbdef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:41:27.288978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-26ebe-8fb564fb5-7z45l to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb112c2d684 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: e0da691d-00b2-4958-9336-424f70e1806b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb12c7b20c6 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 312de6d5-cf7d-4ce9-87f3-0df54ac37cfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26ebe} kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb12de18a4c namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 9ff9bd64-1ebf-4423-aa12-065ee047608a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26ebe} kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'Created container: sequence-graph-26ebe' metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb130c882d1 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: aeaa5494-4711-430f-b6cf-775131ccb710 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26ebe} kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: Started container sequence-graph-26ebe metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb1313a40ab namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 7731c3f0-18ef-4055-a381-18fac3ba2874 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26ebe} kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Stopping container sequence-graph-26ebe metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb36afc0cb7 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 2cf3e56a-d5f7-4533-b218-f986a607d9a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26ebe} kind: Pod name: sequence-graph-26ebe-8fb564fb5-7z45l namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 1797a188-5e80-4a4b-bae4-da7ab3832311 kind: Event lastTimestamp: "2026-04-28T19:42:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:42:04Z" name: sequence-graph-26ebe-8fb564fb5-7z45l.18aa9cb3de9b3256 namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: c180e3c8-69d4-4ad5-b600-3bdfd52c2aa1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-26ebe-8fb564fb5 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: b7d32151-dc17-46aa-bc85-69dda9abff07 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'Created pod: sequence-graph-26ebe-8fb564fb5-7z45l' metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe-8fb564fb5.18aa9cb1123f1cb3 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: c6eb88a8-66fd-4f98-968c-df906e2e26ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: e808e97c-2e12-4fd4-a198-ded1c6efb308 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: Scaled up replica set sequence-graph-26ebe-8fb564fb5 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe.18aa9cb111843346 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 8d11dc88-96d2-45a1-98ce-fd03db951dbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 14ee77c3-a2f1-4d8f-88f3-ef879e04dda9 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: InferenceGraph [sequence-graph-26ebe] is Ready metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe.18aa9cb1158699a0 namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: 8c4ae242-5287-4ba7-aae8-2501815c4924 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 14ee77c3-a2f1-4d8f-88f3-ef879e04dda9 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'Failed to update status for InferenceGraph "sequence-graph-26ebe": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe.18aa9cb117d23ebe namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 2cb88b27-f770-47cc-be6b-7ff04a4fd392 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 14ee77c3-a2f1-4d8f-88f3-ef879e04dda9 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:27Z" name: sequence-graph-26ebe.18aa9cb117d292c6 namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: 76305fd9-09b4-4ff4-848e-b7a9a5293f09 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:32:12.295099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6c7b3-86d869f885-wpbqp to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:32:12Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c2fda8c4e77 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 420744ab-09f4-4f59-a0e6-2651dc668b9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:32:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6c7b3-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:32:12Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c2fe6976c56 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: c7b0dcae-3b83-4d25-87b6-a12618861616 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:32:13Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:32:13Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c30177c0e9a namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: da58913f-d36a-4800-9619-65357a90a4bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6c7b3} kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:32:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:32:13Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c3018a85600 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: d162e5e3-2441-4276-acc9-7ea010778ebd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6c7b3} kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:32:13Z" message: 'Created container: sequence-graph-6c7b3' metadata: creationTimestamp: "2026-04-28T19:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:32:13Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c301bbc4030 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 8946932f-3077-491c-a03e-3248fc4751bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6c7b3} kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:32:13Z" message: Started container sequence-graph-6c7b3 metadata: creationTimestamp: "2026-04-28T19:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:32:13Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9c301c3548e0 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 0d086666-840b-48fc-b5a9-78b6ad19ca05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6c7b3} kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Stopping container sequence-graph-6c7b3 metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9ca3105127b9 namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 694d8589-b6f3-462f-a8cb-e076f4aa7c1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6c7b3} kind: Pod name: sequence-graph-6c7b3-86d869f885-wpbqp namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 28c849df-0a23-4f7a-a897-51577af0df9d kind: Event lastTimestamp: "2026-04-28T19:40:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:54Z" name: sequence-graph-6c7b3-86d869f885-wpbqp.18aa9ca39b76e3bb namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 0c3232f4-7b39-4bf5-beff-4a7802ec7cf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6c7b3-86d869f885 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 84530d33-ce4e-4cf6-810d-2abb2a2c94be kind: Event lastTimestamp: "2026-04-28T19:32:12Z" message: 'Created pod: sequence-graph-6c7b3-86d869f885-wpbqp' metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:12Z" name: sequence-graph-6c7b3-86d869f885.18aa9c2fda23b021 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: bc451c57-7472-46d5-a71c-db99aae8b6df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 6e59205a-34d6-41d5-b8c7-ff22e190f4f9 kind: Event lastTimestamp: "2026-04-28T19:32:12Z" message: Scaled up replica set sequence-graph-6c7b3-86d869f885 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:12Z" name: sequence-graph-6c7b3.18aa9c2fd8f946c2 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: a2787689-dba5-45f1-a0c7-a32af4395969 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: c90fc5ac-9e37-4070-8ad3-ee0e8ab229a2 kind: Event lastTimestamp: "2026-04-28T19:32:12Z" message: InferenceGraph [sequence-graph-6c7b3] is Ready metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:32:12Z" name: sequence-graph-6c7b3.18aa9c2fe0d9ed64 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 84fe1125-ece6-444f-b5f9-49d29b3ae0eb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 0d9d464d-3653-4243-aa85-061b0de6b882 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: sequence-graph-6c7b3.18aa9c3357cdb477 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: f2a29b67-38a6-4f9e-a903-dae0e2c95e58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 0d9d464d-3653-4243-aa85-061b0de6b882 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: sequence-graph-6c7b3.18aa9c335856be32 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 57405695-9622-492b-aa68-bb799bb5e3c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 0d9d464d-3653-4243-aa85-061b0de6b882 kind: Event lastTimestamp: "2026-04-28T19:32:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:42Z" name: sequence-graph-6c7b3.18aa9c36d706f278 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: db2b49a3-6ecc-4f4d-b6ef-6631ac23f212 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 0d9d464d-3653-4243-aa85-061b0de6b882 kind: Event lastTimestamp: "2026-04-28T19:32:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:42Z" name: sequence-graph-6c7b3.18aa9c36d76e75c1 namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: 1823f4ad-df86-4874-8756-9bb97b6e50c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:42:37.580148Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7809f-56768674bd-l46kl to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:42:37Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9cc17070fe0b namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 39a97df9-23d6-401d-bf6c-c0c6a61bdbff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:42:38Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:42:38Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9cc189d1cc9b namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: f3378e5f-1613-4dd8-a150-00cbf45820ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7809f} kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:42:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:42:38Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9cc18b212a1f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 01cdbe01-8c65-45d4-b1e0-5dc62e452f2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7809f} kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:42:38Z" message: 'Created container: sequence-graph-7809f' metadata: creationTimestamp: "2026-04-28T19:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:42:38Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9cc18e006769 namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 54d8dfba-a476-4da8-adc6-c43d05c2adb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7809f} kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:42:38Z" message: Started container sequence-graph-7809f metadata: creationTimestamp: "2026-04-28T19:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:42:38Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9cc18e714c8e namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: f3ee762d-4687-4d6b-8803-6189e94693e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7809f} kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Stopping container sequence-graph-7809f metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9d349c0922d8 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: c9b77551-59d8-4edc-a3aa-ed53ed319644 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7809f} kind: Pod name: sequence-graph-7809f-56768674bd-l46kl namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 8a0de09b-f33b-4bc3-a0d1-00b83d801e34 kind: Event lastTimestamp: "2026-04-28T19:51:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:19Z" name: sequence-graph-7809f-56768674bd-l46kl.18aa9d35264d4d5b namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 2bb279ed-0157-41ac-b906-ae1d2fb018df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7809f-56768674bd namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 4f46d168-0228-4f42-a00f-14eee477f28d kind: Event lastTimestamp: "2026-04-28T19:42:37Z" message: 'Created pod: sequence-graph-7809f-56768674bd-l46kl' metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:37Z" name: sequence-graph-7809f-56768674bd.18aa9cc1701075be namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: e3b37d1b-9d02-47dd-a4aa-02c5286f017c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7809f namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: 7510fc44-840a-4b46-b383-6e6b81f551c6 kind: Event lastTimestamp: "2026-04-28T19:42:37Z" message: Scaled up replica set sequence-graph-7809f-56768674bd from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:37Z" name: sequence-graph-7809f.18aa9cc16f0255ca namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 506bada5-f380-453c-950d-1810a3a4b1ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7809f namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 71fd0007-c00a-4aed-b642-139a0671f2d5 kind: Event lastTimestamp: "2026-04-28T19:42:37Z" message: InferenceGraph [sequence-graph-7809f] is Ready metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:42:37Z" name: sequence-graph-7809f.18aa9cc172e13af5 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: bde606c3-1571-4897-8ebd-516be9845cfb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7809f namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 9ba1f296-5bb3-4025-b2f3-307b7beaec81 kind: Event lastTimestamp: "2026-04-28T19:43:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:43:22Z" name: sequence-graph-7809f.18aa9cc4edb589f5 namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: 3581f700-0b4a-4484-a753-e08512f5d6a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7809f namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 9ba1f296-5bb3-4025-b2f3-307b7beaec81 kind: Event lastTimestamp: "2026-04-28T19:43:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:43:22Z" name: sequence-graph-7809f.18aa9cc4edfc42ac namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 269dd690-22f9-4dc0-a3b9-a5fe55a7f937 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:51:12.816490Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6bad9-5444bd87c8-6slkv to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:51:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:51:12Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d3966eba9a9 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: a6fc1c34-1338-4244-be86-9c0f282c0712 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26020" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:13Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:51:13Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d39802a3a9d namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: c8bc3afd-e662-443a-b6fc-32bfa8b6e83b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6bad9} kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:13Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d39816276d8 namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 6cc3d0c9-9968-4b01-bfc6-9e21d2b81461 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6bad9} kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:13Z" message: 'Created container: splitter-graph-6bad9' metadata: creationTimestamp: "2026-04-28T19:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:13Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d3984642f93 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 2ecd5e08-f5be-419a-9bff-4f45c79c69e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6bad9} kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:13Z" message: Started container splitter-graph-6bad9 metadata: creationTimestamp: "2026-04-28T19:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:13Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d3984dc2c02 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 0ee712e4-4ea1-418d-9e6f-64ef5c47cab3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6bad9} kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: Stopping container splitter-graph-6bad9 metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:22Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d3bbb731e6f namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: d8cd6331-d001-4f7c-9e97-7c106147542d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6bad9} kind: Pod name: splitter-graph-6bad9-5444bd87c8-6slkv namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 29006596-169d-4609-ad97-0a2610da37c1 kind: Event lastTimestamp: "2026-04-28T19:51:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:50Z" name: splitter-graph-6bad9-5444bd87c8-6slkv.18aa9d3c4b6f3c7c namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: fc7ca7da-b710-4882-815c-0b6b008e0550 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6bad9-5444bd87c8 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: ea4e9b53-823c-4764-b923-18b2c0987ab4 kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: 'Created pod: splitter-graph-6bad9-5444bd87c8-6slkv' metadata: creationTimestamp: "2026-04-28T19:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: splitter-graph-6bad9-5444bd87c8.18aa9d39668bdac8 namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 68e5b6a5-b0ba-4531-a14f-30c4f36e360b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 83b8d26c-3d4a-49fd-84e2-29d995767b48 kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: Scaled up replica set splitter-graph-6bad9-5444bd87c8 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: splitter-graph-6bad9.18aa9d3965cab949 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 27c77005-7abe-4242-820a-f58bd193e9c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: 6cf789e9-6732-4b94-8215-c7d0042441c7 kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: InferenceGraph [splitter-graph-6bad9] is Ready metadata: creationTimestamp: "2026-04-28T19:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:12Z" name: splitter-graph-6bad9.18aa9d396987697c namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 2e6dbd5d-7977-4ae8-8891-f84fb2f9a649 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:52:23.036608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-93cee-7d756c9f5c-6gn5j to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:52:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:52:23Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d49c05d9426 namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: b0e63c62-35a9-4268-8a9b-df885712e953 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:23Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T19:52:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-93cee-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:23Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d49ca764cf8 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: c28d8a72-14a2-4e13-986c-1533443c9fcf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:24Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T19:52:24Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:52:24Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d49fd2d4b14 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: f0bf36fb-579e-40a6-a257-7fdab92d9fcd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-93cee} kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T19:52:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:24Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d49feceb42f namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 03fafa28-a25e-44f9-bb6e-5bc65e70c573 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-93cee} kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T19:52:24Z" message: 'Created container: splitter-graph-93cee' metadata: creationTimestamp: "2026-04-28T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:24Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d4a024706ea namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 9a094011-4c73-49a0-98c6-ceab89430980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-93cee} kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T19:52:24Z" message: Started container splitter-graph-93cee metadata: creationTimestamp: "2026-04-28T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:24Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9d4a02c0bf4c namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: fadc037e-0b4e-4a35-bc8e-cd744d96263e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-93cee} kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T20:00:37Z" message: Stopping container splitter-graph-93cee metadata: creationTimestamp: "2026-04-28T20:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:00:37Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9dbcf39cd0f8 namespace: kserve-ci-e2e-test resourceVersion: "30283" uid: 1fcbff36-13c2-4763-a5f3-b435aa88c3f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T20:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-93cee} kind: Pod name: splitter-graph-93cee-7d756c9f5c-6gn5j namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 8ea68c33-7034-4b7d-a6b6-4642a4b344f0 kind: Event lastTimestamp: "2026-04-28T20:01:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T20:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:01:05Z" name: splitter-graph-93cee-7d756c9f5c-6gn5j.18aa9dbd9316eec5 namespace: kserve-ci-e2e-test resourceVersion: "30514" uid: c5eeaab8-25bb-40ed-aa67-1679f9fe73a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-93cee-7d756c9f5c namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: 010386ba-089a-4883-987f-2200db622633 kind: Event lastTimestamp: "2026-04-28T19:52:23Z" message: 'Created pod: splitter-graph-93cee-7d756c9f5c-6gn5j' metadata: creationTimestamp: "2026-04-28T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:23Z" name: splitter-graph-93cee-7d756c9f5c.18aa9d49bffabbcc namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: c973fc6f-aa25-4328-b3af-1ae48e71cdf6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-93cee namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: 1bf36604-c823-4c1f-91af-3ec2bd2c73ae kind: Event lastTimestamp: "2026-04-28T19:52:23Z" message: Scaled up replica set splitter-graph-93cee-7d756c9f5c from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:23Z" name: splitter-graph-93cee.18aa9d49bf2da157 namespace: kserve-ci-e2e-test resourceVersion: "26828" uid: 6a4e8998-5dd7-48f4-a0de-3d28d8ddf634 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-93cee namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 13863e20-9872-433f-a157-8fb96e6d904a kind: Event lastTimestamp: "2026-04-28T19:52:23Z" message: InferenceGraph [splitter-graph-93cee] is Ready metadata: creationTimestamp: "2026-04-28T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:52:23Z" name: splitter-graph-93cee.18aa9d49c2e0ce69 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 4f3e8286-d199-478b-89d7-4b692563c23b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-93cee namespace: kserve-ci-e2e-test resourceVersion: "26826" uid: e9094230-fade-4dcd-a9f6-810e65af4638 kind: Event lastTimestamp: "2026-04-28T19:53:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:53:08Z" name: splitter-graph-93cee.18aa9d4d3db1e98b namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: 1bddb895-2560-448e-ac51-9b7abbfd752e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-93cee namespace: kserve-ci-e2e-test resourceVersion: "26826" uid: e9094230-fade-4dcd-a9f6-810e65af4638 kind: Event lastTimestamp: "2026-04-28T19:53:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:53:08Z" name: splitter-graph-93cee.18aa9d4d3e178dff namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 3a96431a-b09f-4626-940e-c2196633dad2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:30:32.558612Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0600b-predictor-655f9fcfd4-98d2n to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18a1ca4304 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 1ef935a3-0fe9-4b8b-8d95-c0683f9f21e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0600b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18ae4b9627 namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 313339f3-9d8b-4376-ae11-87c6e6e73f8c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18de614a72 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 4a176ee5-e2ad-459f-a149-e92c0729c4f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18dfdae907 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 1c5195b5-4fc9-49fa-a3cd-ddd6e60e242e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18e3031707 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: f65bc120-bdaa-43d1-80bd-1c5bd361b228 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18e3770853 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 53d73981-3542-4977-be3b-43edd7f6bdaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18e38271ad namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 46c9e134-41de-4a1c-abbe-789ecc826883 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18e699f2ed namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 2d1a9854-4480-4d49-9078-facd513ef0a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:30:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:33Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18e7243119 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 11ee8688-98c7-4fa6-a342-b162c97f80e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:39:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:50Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c18fddd653d namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: b95d9ffb-49ae-429f-951f-ba7a2b4e2497 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c99ccfd666e namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: c36c509d-ac86-49dc-b7a7-c406a19d2533 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c99ccfdf53c namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 6050a021-6ebc-43d6-a6b5-dcd5b4960ca2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 26d374c9-c909-4ef7-b723-e234c4404579 kind: Event lastTimestamp: "2026-04-28T19:39:50Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:50Z" name: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n.18aa9c9a7241d179 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 2074f87b-bac2-4ffd-8452-9e557e15a34f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0600b-predictor-655f9fcfd4 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: fedcfe2f-60c3-4093-b22b-65a25d2b35e9 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'Created pod: success-200-isvc-0600b-predictor-655f9fcfd4-98d2n' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b-predictor-655f9fcfd4.18aa9c18a0cedd13 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: a2d7b67b-be7e-43b6-a34e-65eedf2d51c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: a3a1df00-f49d-4ee1-be4d-9411d00b2193 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Scaled up replica set success-200-isvc-0600b-predictor-655f9fcfd4 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b-predictor.18aa9c189c5d1240 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 490a7f2f-f877-4d7b-a3cf-da4b4c438843 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 934ccb8f-ff0c-4a40-98b2-6b5645084ff2 kind: Event lastTimestamp: "2026-04-28T19:32:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:02Z" name: success-200-isvc-0600b-predictor.18aa9c1c1af231c4 namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: fd710a5e-7c14-4e10-9014-9a6efc2af942 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0600b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 934ccb8f-ff0c-4a40-98b2-6b5645084ff2 kind: Event lastTimestamp: "2026-04-28T19:32:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:02Z" name: success-200-isvc-0600b-predictor.18aa9c1c1b501dea namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: c36003ba-9e92-403e-b3fb-938783a3ad7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: 20acccca-50b1-4cbd-8dc8-53af469e998e kind: Event lastTimestamp: "2026-04-28T19:31:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:20Z" name: success-200-isvc-0600b.18aa9c189c1ae0cd namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: dedb1134-916e-43e8-a962-d76d7ff8995a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: 20acccca-50b1-4cbd-8dc8-53af469e998e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-0600b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0600b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b.18aa9c189d2dd5bf namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: ceac99ee-e10a-475f-8613-a427a18938d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: 20acccca-50b1-4cbd-8dc8-53af469e998e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0600b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-0600b.18aa9c189d2e16bb namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: f81e9eb3-d95c-4b07-a2b0-24968b72703b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0600b namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 20acccca-50b1-4cbd-8dc8-53af469e998e kind: Event lastTimestamp: "2026-04-28T19:31:20Z" message: InferenceService [success-200-isvc-0600b] is Ready metadata: creationTimestamp: "2026-04-28T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:20Z" name: success-200-isvc-0600b.18aa9c23b8c5dd85 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: f1a92b56-e997-4786-9fc4-bed5e49c761b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:40:57.655441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0ca59-predictor-b4665bc9b-65wkq to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa2c76eb69 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: be8e7a69-13a2-4680-b428-adf5a436d6b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa46041b72 namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: e050d235-08b8-40a8-8dd6-8b7b53761b9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa473954e7 namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: 302fea00-09ba-4bd6-850f-0cab0eb6ab4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa4a2c28b5 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 9cf80081-c6fc-486e-b74d-f353285e4712 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa4a9e69dc namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: e1a5c437-1f99-4231-8b81-b1a563bba197 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa4aa9310d namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 9c4c1a8e-2cf0-4f2c-a412-25df0db37c1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa4ded4839 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 12b7b7b1-2057-4579-b9c5-db1e90c38662 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:40:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:58Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caa4e601a43 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: dfd8b1cb-0c81-4d20-b8bf-a125a0982bbb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:41:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:36Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9caac3e200f7 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 5426c9cf-066a-43c6-8531-b43619f37fe4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9d2b62ae2018 namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: 6c1ab389-c3cc-4825-a473-32b4389ec33d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 4f191223-363e-4407-ad08-1d395af94d5f kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq.18aa9d2b62aeb64e namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 109556dd-815e-4622-bab9-68ffe720f972 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0ca59-predictor-b4665bc9b namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 0f67d82e-6b51-4a8d-a73b-150c0032b437 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'Created pod: success-200-isvc-0ca59-predictor-b4665bc9b-65wkq' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-0ca59-predictor-b4665bc9b.18aa9caa2c0fabaa namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 5c0d9cf7-ad9c-4cd5-8d15-14b8fc6f52ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 5f7a36e2-639e-4a7f-9ada-95200cd105a8 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Scaled up replica set success-200-isvc-0ca59-predictor-b4665bc9b from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-0ca59-predictor.18aa9caa29d9b2ba namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 8bcbb0bc-7616-48fa-8c8e-fa75e5fddc0c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 8a00a7fe-b7e7-4ee2-9adc-989b33c492f6 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: success-200-isvc-0ca59-predictor.18aa9cada84f2632 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: a455ef40-853a-44cd-b605-33a6edd5c8a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 8a00a7fe-b7e7-4ee2-9adc-989b33c492f6 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: success-200-isvc-0ca59-predictor.18aa9cada985a9e4 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: 46ec40b3-7ffd-41d2-94be-fe9c55f368e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 8a00a7fe-b7e7-4ee2-9adc-989b33c492f6 kind: Event lastTimestamp: "2026-04-28T19:41:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:57Z" name: success-200-isvc-0ca59-predictor.18aa9cb4a71d247d namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: c842ecdf-c4f0-4ffb-a60f-9698f48db151 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ca59-predictor namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 8a00a7fe-b7e7-4ee2-9adc-989b33c492f6 kind: Event lastTimestamp: "2026-04-28T19:41:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:57Z" name: success-200-isvc-0ca59-predictor.18aa9cb4a795ef8b namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: a930fc55-676a-4d78-9313-4fb45998c036 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: a506fc48-c4fa-4925-8a7e-8c742a95cd8d kind: Event lastTimestamp: "2026-04-28T19:41:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:46Z" name: success-200-isvc-0ca59.18aa9caa29a631b4 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 496f3c30-65c5-4b3c-b889-34390d358b7f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: a506fc48-c4fa-4925-8a7e-8c742a95cd8d kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'Failed to update status for InferenceService "success-200-isvc-0ca59": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0ca59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-0ca59.18aa9caa29f52fb9 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 750fcdff-cf50-424c-8f84-b99c08285a31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: a506fc48-c4fa-4925-8a7e-8c742a95cd8d kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0ca59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-0ca59.18aa9caa29f58e36 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 60be122c-ef2d-4ce0-be7b-9b6c12ee7421 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ca59 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a506fc48-c4fa-4925-8a7e-8c742a95cd8d kind: Event lastTimestamp: "2026-04-28T19:41:46Z" message: InferenceService [success-200-isvc-0ca59] is Ready metadata: creationTimestamp: "2026-04-28T19:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:46Z" name: success-200-isvc-0ca59.18aa9cb583ce15f5 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 170abbd2-d8e4-44ee-98f9-243ea9c98072 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:40:27.271175Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-26ebe-predictor-585cfc74d-f52v6 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca3196be0d6 namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 7a06dc0f-90cf-4777-9946-aea4e01cf00d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-26ebe-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca326e3477a namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: aaf356e5-735d-4486-83bd-2f5c8775b56d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca356ae22b1 namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: a5b0fba0-73d4-44fc-aa33-5d94ae936436 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca3580beb4f namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 11ffbd1e-7804-49c3-8ea6-13b8a0273fae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca35b33ad8d namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: 752bae4a-2ee6-4e95-b4da-68569f564f7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca35bad8b6e namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: a1dae409-aacd-453c-9280-b3e3db2efb1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca35bb9a966 namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: ee7aeadd-0e01-4a38-bff3-63bc4ce6a447 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca35ed76ca1 namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: bee2061c-ee1f-4a2c-9e4d-9eeeb33e22a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:40:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:28Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca35f5d9367 namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: 40565e1d-1aee-4284-a402-b343c00db300 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:41:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:05Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9ca3855d8de6 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 4b0ee9bc-27c6-429d-9a96-dd667582aa9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9cb3715132f1 namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 486fb068-c215-459d-91c6-4d36fa39293a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9cb371519af1 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: c979bf33-9a9b-4652-a71e-8c1146f95304 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 10453b2a-d390-4d9c-9088-02ea43478764 kind: Event lastTimestamp: "2026-04-28T19:41:40Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:40Z" name: success-200-isvc-26ebe-predictor-585cfc74d-f52v6.18aa9cb40d7ae359 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 54294c60-7799-430c-8211-9bb897455511 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-26ebe-predictor-585cfc74d namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 0c2e9d42-d268-48dd-bf3e-43f99f7cdb11 kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Created pod: success-200-isvc-26ebe-predictor-585cfc74d-f52v6' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe-predictor-585cfc74d.18aa9ca31904533d namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: db5d49cf-62c1-4425-b277-578f2b2bbe3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20146" uid: a180483d-9a31-4655-80d5-c11440810ddc kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Scaled up replica set success-200-isvc-26ebe-predictor-585cfc74d from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe-predictor.18aa9ca316f62419 namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: 6e8bdb80-0f5d-4bb9-af5a-5b919d591d77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: 310fdd55-7d81-4b8c-831d-aff0d647faa1 kind: Event lastTimestamp: "2026-04-28T19:40:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:42Z" name: success-200-isvc-26ebe-predictor.18aa9ca69551259c namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: d0ee20fd-7abc-49bf-8f57-299928cb5682 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: 310fdd55-7d81-4b8c-831d-aff0d647faa1 kind: Event lastTimestamp: "2026-04-28T19:40:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:42Z" name: success-200-isvc-26ebe-predictor.18aa9ca695b44f99 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: c07e2a4b-3481-466b-9a34-6a43d3ac8a2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 310fdd55-7d81-4b8c-831d-aff0d647faa1 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: success-200-isvc-26ebe-predictor.18aa9caa14617cd5 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: a975669d-520e-4265-b7e0-0aadb2d3cecd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26ebe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 310fdd55-7d81-4b8c-831d-aff0d647faa1 kind: Event lastTimestamp: "2026-04-28T19:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:27Z" name: success-200-isvc-26ebe-predictor.18aa9caa14c8cf0e namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 0ed9f082-570a-45b8-a943-4e154b0ac104 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 7fa0c5bb-8301-4ece-a74c-2cd0e1928cba kind: Event lastTimestamp: "2026-04-28T19:41:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:15Z" name: success-200-isvc-26ebe.18aa9ca316ac505d namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: ba65740d-a397-4bab-9a08-8bb35d175aaf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 7fa0c5bb-8301-4ece-a74c-2cd0e1928cba kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-26ebe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe.18aa9ca3170c9edc namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: ea9b72f1-a184-4bb1-a509-f1840a692fb6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 7fa0c5bb-8301-4ece-a74c-2cd0e1928cba kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-26ebe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-26ebe.18aa9ca3170cfe71 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 8748f5fb-ab48-47ab-a77a-417fae7aa887 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26ebe namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 7fa0c5bb-8301-4ece-a74c-2cd0e1928cba kind: Event lastTimestamp: "2026-04-28T19:41:15Z" message: InferenceService [success-200-isvc-26ebe] is Ready metadata: creationTimestamp: "2026-04-28T19:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:15Z" name: success-200-isvc-26ebe.18aa9cae404f6a0d namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 094152be-bc27-49c6-b5ea-ed59f9aa9d7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:39:47.392504Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2a210-predictor-846fc69544-c4mwm to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c99d077a262 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: 71edd1f8-db60-4f56-9f58-3b135135828d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2a210-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:48Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c99daa391e7 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 2bf29ef3-1255-44c0-8507-2649b1d6ff99 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a4316b6d6 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 4757f419-46b4-4c66-aa20-5e4be28b68dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a446cfa6d namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: e72b95dc-a76b-41f9-bff5-714e1a327357 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a47e60482 namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 52af2c35-6822-4270-aa1d-89c2d60355de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a485f030e namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 4a1164cb-b552-4e06-a16e-f99b0d6950c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a486c3e8f namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: fea0628f-6396-4335-98d8-f499343355e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a4b715779 namespace: kserve-ci-e2e-test resourceVersion: "19808" uid: 6a8d4b38-4c54-4aac-9717-ece1cc94c793 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:39:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:49Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9a4bfd29ca namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 64a86840-ef81-4f25-84be-d7b95279688d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:40:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:26Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9c9aa4518b86 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: d1d54444-27ea-464e-9568-87d4a77d8b81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9caa2a72553b namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: c82eaaab-276c-452d-a1a7-3b9968227736 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a210-predictor-846fc69544-c4mwm namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: dfa7538c-fd88-4bac-ad4b-312c32e20b30 kind: Event lastTimestamp: "2026-04-28T19:40:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:57Z" name: success-200-isvc-2a210-predictor-846fc69544-c4mwm.18aa9caa2a732a0b namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 552ed02f-1c0e-4dce-bef0-08b3122289cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2a210-predictor-846fc69544 namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 42074f24-126a-43f0-9e80-b9fc9c46f128 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'Created pod: success-200-isvc-2a210-predictor-846fc69544-c4mwm' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-2a210-predictor-846fc69544.18aa9c99d01f9656 namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 6fefab0d-9a52-4050-bb32-57534698ab30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: b2ed481d-7dfb-4829-8efe-4124a8ea256e kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Scaled up replica set success-200-isvc-2a210-predictor-846fc69544 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-2a210-predictor.18aa9c99cc613e3f namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: 11f71c08-02e8-48d0-9ad0-2e25e5a8d05d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: cbd2fe85-7c18-4c77-ae26-e1e4d6ff74ae kind: Event lastTimestamp: "2026-04-28T19:40:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:02Z" name: success-200-isvc-2a210-predictor.18aa9c9d4b0a45bb namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: 5ae7eba4-7442-4da8-819c-81d5a3c09960 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: cbd2fe85-7c18-4c77-ae26-e1e4d6ff74ae kind: Event lastTimestamp: "2026-04-28T19:40:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:02Z" name: success-200-isvc-2a210-predictor.18aa9c9d4b50242b namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: f459d79d-837b-4fa7-a519-67c30fb83618 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: cbd2fe85-7c18-4c77-ae26-e1e4d6ff74ae kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: success-200-isvc-2a210-predictor.18aa9ca0ca76bb01 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: ec68291c-609b-4a3d-b3b7-0abf82d1fe05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:40:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a210-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: cbd2fe85-7c18-4c77-ae26-e1e4d6ff74ae kind: Event lastTimestamp: "2026-04-28T19:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:40:47Z" name: success-200-isvc-2a210-predictor.18aa9ca0cadb2581 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 5f91f80c-185c-4ef3-bd5a-44de4f2ad66e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: bc14a339-f779-4b9f-be5f-901bb9c6c141 kind: Event lastTimestamp: "2026-04-28T19:40:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:37Z" name: success-200-isvc-2a210.18aa9c99cc67a85f namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: d3cd01a0-47e0-46f5-b0c2-1bbfc98cd81a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: bc14a339-f779-4b9f-be5f-901bb9c6c141 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-2a210": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2a210": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-2a210.18aa9c99cd03dfce namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: d3a0bec7-d497-4c3d-b403-427f0461dc91 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: bc14a339-f779-4b9f-be5f-901bb9c6c141 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2a210": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:39:47Z" name: success-200-isvc-2a210.18aa9c99cd0477d6 namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: c7631eb3-8372-45b8-9338-acfcfb82f4d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a210 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: bc14a339-f779-4b9f-be5f-901bb9c6c141 kind: Event lastTimestamp: "2026-04-28T19:40:37Z" message: InferenceService [success-200-isvc-2a210] is Ready metadata: creationTimestamp: "2026-04-28T19:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:40:37Z" name: success-200-isvc-2a210.18aa9ca55f73a909 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: e6d1d0b9-0cf0-4607-ad33-ad314dc2b2b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:50:12.700514Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25288" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b67baaf01 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: ca66beaa-5823-4bf8-8ca8-9f8f621c8f67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25306" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b80dc5225 namespace: kserve-ci-e2e-test resourceVersion: "25334" uid: ab0d968b-00fd-48a2-885b-84fb76f55aad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b82399cd1 namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: 34d3a19c-e189-42b3-83ad-4a95c44f1412 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b8595e406 namespace: kserve-ci-e2e-test resourceVersion: "25337" uid: 733d68e0-dda8-4988-a9d4-05ceb9ccf583 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b860764a2 namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 4e2e76f7-5fbb-4f2a-bdd0-a5e27acdc0c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b86127c42 namespace: kserve-ci-e2e-test resourceVersion: "25340" uid: 2cd0c932-ae35-420e-bc67-98b771ffc317 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b8956f41c namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: d048018b-f15f-4ae6-ad38-b289e52f1269 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:13Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2b89c701ae namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: 3126b256-19fb-4392-a4ba-d8b798ce6893 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:50:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:51Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d2bf2bc45cc namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: a9ea7f52-9a18-4a7c-aeaf-9244587de51e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d3bc178a6e5 namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: dab20cb7-d1af-4341-a4d7-9be04e74193b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d3bc1795166 namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: e98eaa10-9f14-479c-8ae8-8bc5788ae40f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: e692a5c3-0746-4dbb-9f52-0baa64cb948f kind: Event lastTimestamp: "2026-04-28T19:51:26Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:26Z" name: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr.18aa9d3c7af3cf22 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: f8fc00a3-aef1-4383-b082-ed0ac0a44762 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6bad9-predictor-5d7df76d56 namespace: kserve-ci-e2e-test resourceVersion: "25265" uid: 4d972d54-42d1-4335-ada4-902a972c2f25 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'Created pod: success-200-isvc-6bad9-predictor-5d7df76d56-h7pdr' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-6bad9-predictor-5d7df76d56.18aa9d2b674d5afc namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: f6ce8105-b846-45a7-8a6a-85003e63d92e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25262" uid: 8ed0819d-a0fe-4188-820f-563562504b4b kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: Scaled up replica set success-200-isvc-6bad9-predictor-5d7df76d56 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-6bad9-predictor.18aa9d2b6340c061 namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: a9578b60-be5a-4a6b-8d93-389c34adc690 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: b7706800-840d-440d-a108-22a072ccfd7f kind: Event lastTimestamp: "2026-04-28T19:50:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:42Z" name: success-200-isvc-6bad9-predictor.18aa9d2ee1c9a26b namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: 5fdf51fe-b19f-422f-9d2b-a20ae310a296 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: b7706800-840d-440d-a108-22a072ccfd7f kind: Event lastTimestamp: "2026-04-28T19:50:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:42Z" name: success-200-isvc-6bad9-predictor.18aa9d2ee232bea8 namespace: kserve-ci-e2e-test resourceVersion: "25594" uid: 00e7d267-cb37-4be2-909c-9ab85c21cff4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: b7706800-840d-440d-a108-22a072ccfd7f kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: success-200-isvc-6bad9-predictor.18aa9d35dfc594e3 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 9cf75444-16b2-4e68-b42f-2b7dcfe00567 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6bad9-predictor namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: b7706800-840d-440d-a108-22a072ccfd7f kind: Event lastTimestamp: "2026-04-28T19:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:12Z" name: success-200-isvc-6bad9-predictor.18aa9d35e02a08f6 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 69db5023-19c3-4665-933f-1c6f90dd951c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: bf1c7f00-4e86-4fa1-bfb4-36ed6b3f91f6 kind: Event lastTimestamp: "2026-04-28T19:51:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:01Z" name: success-200-isvc-6bad9.18aa9d2b631c62f7 namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 02d995c3-2cc9-4ab4-90ea-a2829abb2fa9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: bf1c7f00-4e86-4fa1-bfb4-36ed6b3f91f6 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-6bad9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6bad9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-6bad9.18aa9d2b6371ba76 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 221bb9b2-75ab-4a29-bd6e-f83175529e41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: bf1c7f00-4e86-4fa1-bfb4-36ed6b3f91f6 kind: Event lastTimestamp: "2026-04-28T19:50:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6bad9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:12Z" name: success-200-isvc-6bad9.18aa9d2b637227ca namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 0cc56f87-1224-4297-95b0-d9b759da7a48 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6bad9 namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: bf1c7f00-4e86-4fa1-bfb4-36ed6b3f91f6 kind: Event lastTimestamp: "2026-04-28T19:51:01Z" message: InferenceService [success-200-isvc-6bad9] is Ready metadata: creationTimestamp: "2026-04-28T19:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:01Z" name: success-200-isvc-6bad9.18aa9d36ae3129bb namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: aa1de580-a7e6-43d7-87ad-4ffa2926f44a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:31:12.288671Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c21e1e32972 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: d5bfb7a2-289b-484c-9194-8bd25f2f3b68 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c21fb5c2acc namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 213d7412-0c5e-46c3-a29a-73e8b985e28d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c21fd0e78aa namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: ffcda07a-e81f-4313-ad49-08df92c1b37b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c2201901db2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 6f9ecb8d-eee6-44b8-b025-b80701b55434 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c220207bf7e namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 55052654-5b72-4e5c-87d1-23a7838c0ace reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c22021497bf namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 9ca22bcd-5fb5-499a-a029-432709dffccf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c2204e0d674 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 14a992ee-3751-4c14-a0f4-6b2d8a711379 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c2205547659 namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: eedc41f3-83ce-4909-8f90-93e46a4dbf65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:40:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:30Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9c2256bb1021 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 32da0f7b-6488-4089-b379-33ee7895fcde reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9ca316b6dceb namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: b98a37e0-ed42-45e0-8281-5e6670656bd0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:40:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:27Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9ca316b75568 namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: 8a03a90d-e7dd-4fb8-9ba3-a4fcabbc5d5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: a2d82b36-d9ec-4822-aae8-5b166edea3ef kind: Event lastTimestamp: "2026-04-28T19:40:30Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:30Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c.18aa9ca3cb1d430e namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 2225dd64-1ee8-4df6-aa3c-d684d970ef55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6c7b3-predictor-7f4499fcf8 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 3a37b838-3157-4f6d-80c9-804583f207b3 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created pod: success-200-isvc-6c7b3-predictor-7f4499fcf8-f5g4c' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor-7f4499fcf8.18aa9c21e17e5f3b namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: d9e4a5cd-8797-4efd-a3d6-e446f62d078f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 17a5a030-1260-44f1-81ba-980d99380c9e kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Scaled up replica set success-200-isvc-6c7b3-predictor-7f4499fcf8 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:12Z" name: success-200-isvc-6c7b3-predictor.18aa9c21df5669b6 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: d3f8dd63-c6a9-47c1-89be-a3076154944a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 53ea3068-0427-41af-9375-5faa483f8899 kind: Event lastTimestamp: "2026-04-28T19:31:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:42Z" name: success-200-isvc-6c7b3-predictor.18aa9c255e0663dc namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 2a9de9c1-fd2c-41d6-bfcb-7d9f6032d25f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 53ea3068-0427-41af-9375-5faa483f8899 kind: Event lastTimestamp: "2026-04-28T19:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:42Z" name: success-200-isvc-6c7b3-predictor.18aa9c255e61860b namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 1590d468-9cdc-4aee-ad0e-146c003fd171 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 53ea3068-0427-41af-9375-5faa483f8899 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: success-200-isvc-6c7b3-predictor.18aa9c2c5c3978a6 namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: 7a4c4270-5425-48f7-9863-177275002e03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c7b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 53ea3068-0427-41af-9375-5faa483f8899 kind: Event lastTimestamp: "2026-04-28T19:32:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:27Z" name: success-200-isvc-6c7b3-predictor.18aa9c2c5c89d44f namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: c5610ea6-fc60-4d66-b0de-288f23f700ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: d78b1dc7-071b-49c9-8cc3-0b12a02ed550 kind: Event lastTimestamp: "2026-04-28T19:32:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:32:00Z" name: success-200-isvc-6c7b3.18aa9c21df55b23f namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: c510b3bc-c2d2-40d3-9273-9da97dda42a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:32:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c7b3 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d78b1dc7-071b-49c9-8cc3-0b12a02ed550 kind: Event lastTimestamp: "2026-04-28T19:32:00Z" message: InferenceService [success-200-isvc-6c7b3] is Ready metadata: creationTimestamp: "2026-04-28T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:32:00Z" name: success-200-isvc-6c7b3.18aa9c2d1192a7de namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: f8abff06-159c-4ee1-9436-a3da80cc0a9f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:41:37.494794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7809f-predictor-b4788b9f6-r48p7 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb373134621 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: de0cff2e-3abb-4dde-adc5-5b84bed0968f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7809f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb384fc3d0d namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ed42de2e-f676-4a1d-973b-39151f322034 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b0ce0563 namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: c83de2c9-c754-485b-ab1e-3aadf7095e6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b21156fc namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 6bbea097-51e5-4c85-a088-3675133af7be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b51d2006 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: a94c6e25-d83d-482e-b66a-c385d3015a62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b592a7cf namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: bb8ea2d5-3f52-4a34-9139-da53c6bf3c44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b59f38e7 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: aae69247-7bea-4653-b056-5972409ae18a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b8a2b890 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 7d4f483f-002b-4ed2-b460-58558df41b3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:41:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:41:38Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3b91d8e2c namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 1d206d22-6b1f-4c03-8f33-35a255fa9645 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:50:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:55Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9cb3e0b2b8b6 namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: ebf1d320-e520-4c51-8186-5abf87238b1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9d34a5779c27 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 88294c84-189f-45da-a2ec-780788a5a73d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9d34a578239a namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 01b47bf6-ef12-4157-80a3-b1368d66f892 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 140418b4-b40e-44c3-8c66-f09c82585534 kind: Event lastTimestamp: "2026-04-28T19:50:55Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:55Z" name: success-200-isvc-7809f-predictor-b4788b9f6-r48p7.18aa9d3555047759 namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 56cd2153-cdbb-4107-aaf9-87393f4a1f7f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7809f-predictor-b4788b9f6 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: e22c5403-8c02-4b26-8d12-6ab3f70b8134 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'Created pod: success-200-isvc-7809f-predictor-b4788b9f6-r48p7' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f-predictor-b4788b9f6.18aa9cb372a9fd3b namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 7abc202d-674a-48d7-a6e4-9cb06ec61e29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: 80ca7552-025d-47cc-b4e5-e8b65d9c58f3 kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: Scaled up replica set success-200-isvc-7809f-predictor-b4788b9f6 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f-predictor.18aa9cb370ae05f5 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 64b1768e-45fa-433d-9a8a-53d55abf9106 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 09d64e9a-ef93-4c78-82d0-624674bf9a39 kind: Event lastTimestamp: "2026-04-28T19:42:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:22Z" name: success-200-isvc-7809f-predictor.18aa9cb6f0063a51 namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: e198685c-cec8-4fe7-b25f-e105e55b1447 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 09d64e9a-ef93-4c78-82d0-624674bf9a39 kind: Event lastTimestamp: "2026-04-28T19:42:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:22Z" name: success-200-isvc-7809f-predictor.18aa9cb6f084716f namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 6aa3ab98-d061-49c3-b28e-d563a9fad32a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:42:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 09d64e9a-ef93-4c78-82d0-624674bf9a39 kind: Event lastTimestamp: "2026-04-28T19:42:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:52Z" name: success-200-isvc-7809f-predictor.18aa9cc16d160246 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 9a998358-3248-4f84-9e83-f8107cca14af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:42:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7809f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 09d64e9a-ef93-4c78-82d0-624674bf9a39 kind: Event lastTimestamp: "2026-04-28T19:42:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:42:52Z" name: success-200-isvc-7809f-predictor.18aa9cc16d7cfa4a namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: 8c4f17f9-7942-418b-998c-472c7b2fcd2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 50c4c6e8-7a34-42e8-9d98-3a220d4a2f4d kind: Event lastTimestamp: "2026-04-28T19:42:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:42:25Z" name: success-200-isvc-7809f.18aa9cb370b764fe namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 51fdf360-fd55-4b4d-a46c-9580eac61a4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 50c4c6e8-7a34-42e8-9d98-3a220d4a2f4d kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-7809f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7809f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f.18aa9cb37135b724 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 8a807cd0-6a39-4e0e-ae4d-35df07bedccc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 50c4c6e8-7a34-42e8-9d98-3a220d4a2f4d kind: Event lastTimestamp: "2026-04-28T19:41:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7809f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:41:37Z" name: success-200-isvc-7809f.18aa9cb371361070 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 2adfa96c-cc9c-4372-a761-032efb143d54 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:42:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7809f namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 50c4c6e8-7a34-42e8-9d98-3a220d4a2f4d kind: Event lastTimestamp: "2026-04-28T19:42:25Z" message: InferenceService [success-200-isvc-7809f] is Ready metadata: creationTimestamp: "2026-04-28T19:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:42:25Z" name: success-200-isvc-7809f.18aa9cbe9b8119e1 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 84bd4c06-be32-4d44-b4b8-0d77cb109a8c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:51:22.968868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-93cee-predictor-7d9b86d498-48dr9 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3bc40ca898 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 7f245521-9e86-49b5-8c58-3862b998a00a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3bdd876451 namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 98287081-fdb3-4e5d-a901-ab2101919766 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3bdec3cb15 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: ca44012e-ebe0-4aaf-848c-307665d88ce1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3be284b4b8 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: d17d4d13-00bc-41e6-a687-d7aa5e4a6c7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3be2f9232b namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 57bc7f8d-a7c0-4682-8913-06f9856d6fea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3be3061561 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 3d8d8334-ce00-4368-b61d-f14bc8f67afb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3be5f1e60c namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 5d185137-6b7a-49eb-9dc2-8e1555028975 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:51:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:23Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3be6e9e6d5 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: 1cd9a6f2-2c4b-4670-ac34-aa219ead4b3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T19:52:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:52:01Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9d3c4dbe94a4 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: d0620417-0f5f-48ac-903a-982ca689246a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T20:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T20:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:00:37Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9dbcfa1399fa namespace: kserve-ci-e2e-test resourceVersion: "30298" uid: 140ed64c-bc7a-4baf-aaae-0995caf0f436 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: a3c065ac-d90a-42d8-86fd-49d138e4cde4 kind: Event lastTimestamp: "2026-04-28T20:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T20:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:00:37Z" name: success-200-isvc-93cee-predictor-7d9b86d498-48dr9.18aa9dbcfa14239b namespace: kserve-ci-e2e-test resourceVersion: "30300" uid: 527dc757-aed3-46a4-9b8f-c2c342fd80f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-93cee-predictor-7d9b86d498 namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: 1c00a763-5fc0-47e4-8baa-3bb8607428cb kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: 'Created pod: success-200-isvc-93cee-predictor-7d9b86d498-48dr9' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-93cee-predictor-7d9b86d498.18aa9d3bc32eb274 namespace: kserve-ci-e2e-test resourceVersion: "26178" uid: 0b1ad660-1bba-4fbe-b498-b062f1d53310 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: afd5cace-1955-45bf-bcb0-85c995c419bf kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: Scaled up replica set success-200-isvc-93cee-predictor-7d9b86d498 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-93cee-predictor.18aa9d3bc173b7b4 namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: 5f899873-7bc5-4e56-9b21-2f98de9d168f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: c51474cc-166f-4953-aa1c-a08f70149985 kind: Event lastTimestamp: "2026-04-28T19:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:07Z" name: success-200-isvc-93cee-predictor.18aa9d3f40227b28 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 5cc23d0e-d46c-4b43-b66b-611ff64be24a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: c51474cc-166f-4953-aa1c-a08f70149985 kind: Event lastTimestamp: "2026-04-28T19:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:07Z" name: success-200-isvc-93cee-predictor.18aa9d3f40882d66 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 741ecd7d-2383-4909-b4c7-66193cb1e5a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: c51474cc-166f-4953-aa1c-a08f70149985 kind: Event lastTimestamp: "2026-04-28T19:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:37Z" name: success-200-isvc-93cee-predictor.18aa9d49bcaff139 namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: b0b626a5-b94c-43f6-9df9-b0bde7fa4c0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93cee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: c51474cc-166f-4953-aa1c-a08f70149985 kind: Event lastTimestamp: "2026-04-28T19:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:37Z" name: success-200-isvc-93cee-predictor.18aa9d49bd0fd762 namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: ca54e80f-4f21-40b3-ac70-82337ed8b182 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 1ae69d65-21e1-4c08-a7c9-802c51085c32 kind: Event lastTimestamp: "2026-04-28T19:52:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:52:11Z" name: success-200-isvc-93cee.18aa9d3bc17e2530 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: 44c35806-f3fc-49b0-9d55-e11e012f5a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 1ae69d65-21e1-4c08-a7c9-802c51085c32 kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: 'Failed to update status for InferenceService "success-200-isvc-93cee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-93cee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-93cee.18aa9d3bc1f2a4f9 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 422e8048-19b0-4020-b2cf-8c5fc7390aa5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 1ae69d65-21e1-4c08-a7c9-802c51085c32 kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-93cee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-93cee.18aa9d3bc1f3291a namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: eeb52608-2cb7-4b3c-a35c-a4943c7fdb4a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93cee namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: 1ae69d65-21e1-4c08-a7c9-802c51085c32 kind: Event lastTimestamp: "2026-04-28T19:52:11Z" message: InferenceService [success-200-isvc-93cee] is Ready metadata: creationTimestamp: "2026-04-28T19:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:52:11Z" name: success-200-isvc-93cee.18aa9d4708e6a72f namespace: kserve-ci-e2e-test resourceVersion: "26717" uid: 0ebdf831-6753-468b-9224-45ea22b809db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:01.967053Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e1324-predictor-7798b8b7c8-cktzl to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:01Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c038a1cc510 namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: dffbc1c8-d748-4373-a68a-199c486ee56c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:02Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c03a3ba5ee5 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 2bafce55-1f55-4d11-8d04-d681007faf93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:02Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c03a51d8619 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: f7e718c6-e446-4600-90b7-a2052a30c97d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" in 13.405s (13.405s including waiting). Image size: 1335719555 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c06c4245b7d namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 5d36bd16-b2a5-47ae-ba04-7a1a989b92f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c06c78e313b namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 18f7e5b5-8753-44e6-80ea-52ff8e71938a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c06c8202470 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 035f249b-a60f-4e5b-ae20-6f4a30a359d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-28T19:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:15Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c06c82fd77f namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 4184ac23-93c8-4b6b-bd13-12fad8c93f5b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.573s (2.573s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:18Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c07619b07d6 namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 404fb3b5-c77f-4e57-94bc-1b81308f0f2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:18Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c076489493f namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 44d79cfe-c361-4083-92db-2510a18053e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:18Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c076502bd9d namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: ff6c477c-0446-4221-b935-5ac7b71ba12a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:29:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:55Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c07b3bf81ec namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: d963af77-e8e5-4e73-bba8-2028b4c9a5f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c1898ace567 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 55ae0f9a-51ef-4c9c-b184-0f8687dd8a13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: ba0d6624-ce60-4b4c-ab0d-c60cddbbddcf kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl.18aa9c1898ad78f7 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 3ec9e177-cd1e-427c-9819-1d666b93a183 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e1324-predictor-7798b8b7c8 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 12762c06-1b57-45bf-9bd0-43353f37e223 kind: Event lastTimestamp: "2026-04-28T19:29:01Z" message: 'Created pod: success-200-isvc-e1324-predictor-7798b8b7c8-cktzl' metadata: creationTimestamp: "2026-04-28T19:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:01Z" name: success-200-isvc-e1324-predictor-7798b8b7c8.18aa9c0389a26551 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: f9ed02ae-c1a0-450c-aee1-04839343bb43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 5d281fba-498a-473c-b241-2f16fb2f294f kind: Event lastTimestamp: "2026-04-28T19:29:01Z" message: Scaled up replica set success-200-isvc-e1324-predictor-7798b8b7c8 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:01Z" name: success-200-isvc-e1324-predictor.18aa9c0385088def namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: fb6344d5-0ad8-4dc5-a76d-06ad6085d53c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: bc540caa-6cdb-4d00-85a1-4ef13834b44e kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:47Z" name: success-200-isvc-e1324-predictor.18aa9c070a908170 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: f9084dae-ef28-4ca5-88fa-478af68458bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: bc540caa-6cdb-4d00-85a1-4ef13834b44e kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:47Z" name: success-200-isvc-e1324-predictor.18aa9c070b926c4c namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: a45d3e6e-eda6-4cd5-94b2-3e5652811a8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: bc540caa-6cdb-4d00-85a1-4ef13834b44e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-e1324-predictor.18aa9c1187d48d39 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: 23eefa33-fa0c-42c8-8bb9-c064cf0ccce7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1324-predictor namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: bc540caa-6cdb-4d00-85a1-4ef13834b44e kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:32Z" name: success-200-isvc-e1324-predictor.18aa9c11883dee5a namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 8e1dd08a-9e6c-4cd8-8387-512cd90038e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 7d552f26-7adc-4832-a428-0b7bf9c96b14 kind: Event lastTimestamp: "2026-04-28T19:30:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:05Z" name: success-200-isvc-e1324.18aa9c038b543a17 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: fc045ea2-cf91-41c4-a0ab-8a707a2d8284 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 7d552f26-7adc-4832-a428-0b7bf9c96b14 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-e1324": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e1324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:02Z" name: success-200-isvc-e1324.18aa9c0391cc9c25 namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 2ee69db6-6de2-43b3-9be2-9f1d9ab3f73f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 7d552f26-7adc-4832-a428-0b7bf9c96b14 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e1324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:02Z" name: success-200-isvc-e1324.18aa9c0391cced18 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 14c9c33c-797e-45d4-a37d-b5d237213680 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1324 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 7d552f26-7adc-4832-a428-0b7bf9c96b14 kind: Event lastTimestamp: "2026-04-28T19:30:05Z" message: InferenceService [success-200-isvc-e1324] is Ready metadata: creationTimestamp: "2026-04-28T19:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:05Z" name: success-200-isvc-e1324.18aa9c126eae2855 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 2ad358be-0060-4e11-b443-7f3d768cde22 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:50:52.415861Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e5e5a-predictor-7bd5747694-zwckw to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34a6f2c979 namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 6694d762-e0ba-4090-8fd8-f7d8dd6c44c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c03f6d0e namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: 662b7a69-0fd5-45cb-bb9d-231f1a86ebd0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c185ee29 namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 2fcf5f25-0c2e-44d7-84cd-b4b55a92d143 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c4a1bd3d namespace: kserve-ci-e2e-test resourceVersion: "25779" uid: 98e32490-199c-4619-9098-9641f0969151 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c519a1b8 namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: c19de260-def2-411e-bf55-ded4cf3ba1a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c5274102 namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: e0ade97b-25ec-4584-ad38-9e9e86dbd6ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c90798b4 namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 53027c92-c824-47ed-8d40-1c4c4b98618c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d34c986f252 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: 22579136-60c4-42a5-8470-91c4ae30c18a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T19:51:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:30Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9d350f4050db namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 825421f6-2d06-4493-b61e-0d7db1575930 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T20:08:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T20:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:11Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9e26ae3051b5 namespace: kserve-ci-e2e-test resourceVersion: "33441" uid: 76b1be03-b019-4b5e-abc4-4b2e1f2b3d95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: dfc04099-98ac-4199-beeb-90bdf50f0477 kind: Event lastTimestamp: "2026-04-28T20:08:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T20:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:11Z" name: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw.18aa9e26ae30dda8 namespace: kserve-ci-e2e-test resourceVersion: "33442" uid: 3eb0b517-f8d1-4b4f-afca-a33b14a8fc2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e5e5a-predictor-7bd5747694 namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: f3b290a2-d0b8-4b81-8a10-a1028901226d kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Created pod: success-200-isvc-e5e5a-predictor-7bd5747694-zwckw' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor-7bd5747694.18aa9d34a6572f21 namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: bc8a6482-b3eb-461d-87ae-58f454852081 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 91f20221-3acc-400f-a469-c3b0d1cac3e3 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: Scaled up replica set success-200-isvc-e5e5a-predictor-7bd5747694 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a-predictor.18aa9d34a4053707 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: b68af8ea-248f-40a3-8965-6927959f549e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:51:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 72bd8c04-d959-42be-8355-edbc0b4f0205 kind: Event lastTimestamp: "2026-04-28T19:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:07Z" name: success-200-isvc-e5e5a-predictor.18aa9d3822a24ed9 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: c0cdc9f0-f743-42f6-a294-923f4f7761bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:51:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 72bd8c04-d959-42be-8355-edbc0b4f0205 kind: Event lastTimestamp: "2026-04-28T19:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:07Z" name: success-200-isvc-e5e5a-predictor.18aa9d3822ffc2a2 namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 92d6f4a5-6f88-4fd5-a789-e2755be3d605 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: 72bd8c04-d959-42be-8355-edbc0b4f0205 kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-e5e5a-predictor.18aa9d3ba24f83c5 namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 015a6534-d62e-4f00-b44e-32a3be7b6a09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5e5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: 72bd8c04-d959-42be-8355-edbc0b4f0205 kind: Event lastTimestamp: "2026-04-28T19:51:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:22Z" name: success-200-isvc-e5e5a-predictor.18aa9d3ba2f5b24f namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 75727768-19eb-49a2-865b-183923dc2120 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 7ae3091b-d00a-411d-a27c-b2bef982cdb3 kind: Event lastTimestamp: "2026-04-28T19:51:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:40Z" name: success-200-isvc-e5e5a.18aa9d34a3f821b9 namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: 8b415c15-c0fc-4a6c-88cb-109dba865650 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 7ae3091b-d00a-411d-a27c-b2bef982cdb3 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-e5e5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e5e5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a.18aa9d34a494ea35 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: e8cedd7d-edcb-4be2-9829-caaa99349752 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:50:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 7ae3091b-d00a-411d-a27c-b2bef982cdb3 kind: Event lastTimestamp: "2026-04-28T19:50:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e5e5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:50:52Z" name: success-200-isvc-e5e5a.18aa9d34a495450b namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 3555b3dd-9ae0-479d-872e-81ffdf2bb75c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 7ae3091b-d00a-411d-a27c-b2bef982cdb3 kind: Event lastTimestamp: "2026-04-28T19:51:40Z" message: InferenceService [success-200-isvc-e5e5a] is Ready metadata: creationTimestamp: "2026-04-28T19:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:40Z" name: success-200-isvc-e5e5a.18aa9d3fca1a20b4 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: b33a8e6f-9c59-4918-a3e6-c2b5695dff2d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:31:32.518701Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0600b-587b488647-lbj9j to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c2697b0a106 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49a2e626-aee3-4f21-a750-0c7af36c08f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c26b0b94e81 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 3801799a-72f6-4055-881f-48a68896e207 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0600b} kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c26b1e52d55 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 0aab663e-16f2-4039-a8e0-3247368a156a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0600b} kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:31:33Z" message: 'Created container: switch-graph-0600b' metadata: creationTimestamp: "2026-04-28T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:33Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c26b4c6262a namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 855f80c2-9265-49f5-b507-d7e40c2435c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0600b} kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:31:33Z" message: Started container switch-graph-0600b metadata: creationTimestamp: "2026-04-28T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:33Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c26b533f2b3 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 33a7281c-fae6-4fa9-9e00-4f637774363c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0600b} kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:39:47Z" message: Stopping container switch-graph-0600b metadata: creationTimestamp: "2026-04-28T19:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:39:47Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c99c54d3994 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: bd276bf3-3b1f-4e98-b51c-557408d00218 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0600b} kind: Pod name: switch-graph-0600b-587b488647-lbj9j namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 5f9e1cee-58d5-48e7-a221-5d22c91e1a32 kind: Event lastTimestamp: "2026-04-28T19:40:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:40:14Z" name: switch-graph-0600b-587b488647-lbj9j.18aa9c9a438dbbac namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: 6da23e20-4540-4a03-ab73-222a9fea3394 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0600b-587b488647 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: c2bd5136-b124-4fa9-846b-62191c52eed1 kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: 'Created pod: switch-graph-0600b-587b488647-lbj9j' metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b-587b488647.18aa9c269745d3c7 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: c05aedbf-7886-468a-880a-73a087c730f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0600b namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: fbba4a63-6089-469d-a83a-b3076c5daddb kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: Scaled up replica set switch-graph-0600b-587b488647 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b.18aa9c269634ef95 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: ecfe53f7-a769-409f-ad9a-aba972f1aa25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0600b namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 9fa51570-2dbe-4176-ba7e-61cae7612d6e kind: Event lastTimestamp: "2026-04-28T19:31:32Z" message: InferenceGraph [switch-graph-0600b] is Ready metadata: creationTimestamp: "2026-04-28T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:32Z" name: switch-graph-0600b.18aa9c2699b10c17 namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: e4deee10-5627-4941-8e77-f2b9d59a2770 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0600b namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 30eba4f8-d68f-4684-bd38-1ba90102544b kind: Event lastTimestamp: "2026-04-28T19:32:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:17Z" name: switch-graph-0600b.18aa9c2a152d5ec2 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: b63f2c73-702b-4503-9ec5-0d0c47f958ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0600b namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 30eba4f8-d68f-4684-bd38-1ba90102544b kind: Event lastTimestamp: "2026-04-28T19:32:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:32:17Z" name: switch-graph-0600b.18aa9c2a156c7323 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: 8117d4a2-03bf-46eb-a3ad-c5cc4065c72b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:30:22.058470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e1324-b8c5ffc4-w28k7 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:30:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:30:22Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c162feeda4b namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 9f15b9eb-9a85-42da-9e93-c184e035e0ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-28T19:30:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e1324-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:47Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c163d038630 namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: bb573817-7086-4056-8544-2a89579510fb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:23Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:30:23Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c166d6bc493 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: cb615334-12ce-43e2-bd28-6152b469e2e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:23Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" metadata: creationTimestamp: "2026-04-28T19:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:23Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c166e95ab9f namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 428fabd2-a6d4-4931-b4ee-40e07e5dba8d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" in 2.666s (2.666s including waiting). Image size: 216346748 bytes.' metadata: creationTimestamp: "2026-04-28T19:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:25Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c170d8b5d30 namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: f435e5e1-46ef-47f7-bccf-0cdf2c78f001 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:25Z" message: 'Created container: switch-graph-e1324' metadata: creationTimestamp: "2026-04-28T19:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:25Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c1710a32829 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: bb2e4de4-2a29-419d-a17f-75b7802b8d8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:25Z" message: Started container switch-graph-e1324 metadata: creationTimestamp: "2026-04-28T19:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:25Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c171124eee5 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 310f8eaa-8096-443f-b538-61b96048569d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:30:32Z" message: Stopping container switch-graph-e1324 metadata: creationTimestamp: "2026-04-28T19:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:30:32Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c1890a7aa46 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 2207bd47-a69e-4760-b7ff-38dcac6c04a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e1324} kind: Pod name: switch-graph-e1324-b8c5ffc4-w28k7 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: c35727eb-b64c-43fb-814e-4c7bf7a7bd01 kind: Event lastTimestamp: "2026-04-28T19:31:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:02Z" name: switch-graph-e1324-b8c5ffc4-w28k7.18aa9c19af36dbe7 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: fd95a582-a2db-493c-9ae3-2df94e2b8352 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e1324-b8c5ffc4 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 6c883568-1ea8-4954-b579-d6b909c08534 kind: Event lastTimestamp: "2026-04-28T19:30:22Z" message: 'Created pod: switch-graph-e1324-b8c5ffc4-w28k7' metadata: creationTimestamp: "2026-04-28T19:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:22Z" name: switch-graph-e1324-b8c5ffc4.18aa9c162f938f4a namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: a9023d53-f0ba-4e29-b9c4-de21e7267b59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e1324 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: da2fb559-8b75-4cfc-ad03-0ad43fa6c7c5 kind: Event lastTimestamp: "2026-04-28T19:30:22Z" message: Scaled up replica set switch-graph-e1324-b8c5ffc4 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:22Z" name: switch-graph-e1324.18aa9c162e89d1dd namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 817a8954-00f1-46f5-ab16-3215fba1d070 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e1324 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: a112d54f-9aca-4b31-9b84-2835a7845af1 kind: Event lastTimestamp: "2026-04-28T19:30:22Z" message: InferenceGraph [switch-graph-e1324] is Ready metadata: creationTimestamp: "2026-04-28T19:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:22Z" name: switch-graph-e1324.18aa9c16326aa669 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 33fcc640-0361-4aab-8db8-5278019f6e8b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:51:52.487199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e5e5a-65c9566545-brp96 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-28T19:51:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:51:52Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9d42a37a9a36 namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: bb4dcddc-9939-42b2-983a-4c619033342b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4f94975c-hk6df source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T19:51:53Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:51:53Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9d42c9a95ec1 namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: c380036e-78ea-4bac-a5cd-d584414a2248 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e5e5a} kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T19:51:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:53Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9d42caed1862 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 87b8e0d8-7ac4-4cc6-bec0-dfe465b85860 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e5e5a} kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T19:51:53Z" message: 'Created container: switch-graph-e5e5a' metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:53Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9d42cdd7ce26 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: e5e3daa2-5b1d-4f05-bd33-afe109eb06b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e5e5a} kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T19:51:53Z" message: Started container switch-graph-e5e5a metadata: creationTimestamp: "2026-04-28T19:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:51:53Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9d42ce52c12c namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: 5409c915-cd8f-4b44-930f-1c686f5ef898 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T20:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e5e5a} kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T20:08:11Z" message: Stopping container switch-graph-e5e5a metadata: creationTimestamp: "2026-04-28T20:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:11Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9e26a56c3ff0 namespace: kserve-ci-e2e-test resourceVersion: "33423" uid: c7d020c0-0dfd-4dde-b391-245e8732577d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T20:08:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e5e5a} kind: Pod name: switch-graph-e5e5a-65c9566545-brp96 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 44f4ad2a-fae1-4e8e-932b-b7d20fb0eb60 kind: Event lastTimestamp: "2026-04-28T20:08:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T20:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T20:08:34Z" name: switch-graph-e5e5a-65c9566545-brp96.18aa9e274139b477 namespace: kserve-ci-e2e-test resourceVersion: "33662" uid: 1e82a3e9-cd70-452e-a4e2-43f8561690c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e5e5a-65c9566545 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 6e5aebf6-3649-42b3-9e5a-3bf83300eee1 kind: Event lastTimestamp: "2026-04-28T19:51:52Z" message: 'Created pod: switch-graph-e5e5a-65c9566545-brp96' metadata: creationTimestamp: "2026-04-28T19:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:52Z" name: switch-graph-e5e5a-65c9566545.18aa9d42a3078643 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: e24050c8-ed45-4e4a-be5c-73a38886615f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 91c90ee4-1dba-41f4-9228-580fd95688e9 kind: Event lastTimestamp: "2026-04-28T19:51:52Z" message: Scaled up replica set switch-graph-e5e5a-65c9566545 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:51:52Z" name: switch-graph-e5e5a.18aa9d42a264d0e9 namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: ad411e4b-f1d3-4521-8b31-4d3978cd78af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:51:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: be1871b9-86d5-4f9c-a6eb-fd1ad41971df kind: Event lastTimestamp: "2026-04-28T19:51:52Z" message: InferenceGraph [switch-graph-e5e5a] is Ready metadata: creationTimestamp: "2026-04-28T19:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:51:52Z" name: switch-graph-e5e5a.18aa9d42a8658363 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 31d8e94b-1e77-4a09-b704-df555cf5230a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 3774beb1-9cee-4879-8f33-9db827f72535 kind: Event lastTimestamp: "2026-04-28T19:52:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:37Z" name: switch-graph-e5e5a.18aa9d4620f79d8c namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: f6088a35-2b0a-4d79-bfe1-0b41fc239ab0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e5e5a namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 3774beb1-9cee-4879-8f33-9db827f72535 kind: Event lastTimestamp: "2026-04-28T19:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:52:37Z" name: switch-graph-e5e5a.18aa9d46215da4c1 namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: df1f0252-2222-4621-be07-b8c70ea38818 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33677"