--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:58:37.560382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-510ff-85d6b4bb57-jr287 to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:58:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:58:37Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c1c53124f namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: ddcca8fd-23c4-4e13-bfb0-0075a807d10b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:37Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-510ff-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:37Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c29fd7b67 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 48d487e8-c04b-4687-a378-091d8550db60 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:38Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T16:58:38Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:58:38Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c595476b6 namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: 8642d60a-82ff-4ba6-8e4a-711bd2c2408b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-510ff} kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T16:58:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:38Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c5ab3cb60 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 39159b93-8951-407b-9edc-52ac73f41d37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-510ff} kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T16:58:38Z" message: 'Created container: ensemble-graph-510ff' metadata: creationTimestamp: "2026-04-24T16:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:38Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c5daa5e4c namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 113c8d8e-860f-4c91-aee8-929065b34d92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-510ff} kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T16:58:38Z" message: Started container ensemble-graph-510ff metadata: creationTimestamp: "2026-04-24T16:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:38Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a9597c5e1fbc6b namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: d3d9d737-cdd2-4868-b802-ebe111326979 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-510ff} kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Stopping container ensemble-graph-510ff metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a959ef3f9a31be namespace: kserve-ci-e2e-test resourceVersion: "23370" uid: 98bafed8-32a6-425c-b7f8-9b156e4c314d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-510ff} kind: Pod name: ensemble-graph-510ff-85d6b4bb57-jr287 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 3238c417-fa6b-4acd-be0a-ae1791599d78 kind: Event lastTimestamp: "2026-04-24T17:07:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T17:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:19Z" name: ensemble-graph-510ff-85d6b4bb57-jr287.18a959efe509f380 namespace: kserve-ci-e2e-test resourceVersion: "23706" uid: 09b6cd39-98bb-49ff-a0e0-9899239bc524 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-510ff-85d6b4bb57 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 5919f366-915f-43ad-ab84-bc62f9830239 kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'Created pod: ensemble-graph-510ff-85d6b4bb57-jr287' metadata: creationTimestamp: "2026-04-24T16:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: ensemble-graph-510ff-85d6b4bb57.18a9597c1bdb4958 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: d0b475dc-0e60-4ff1-96ad-1074460f10c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-510ff namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: e8414ab6-8e4c-4f8f-810b-db8e084f9dbb kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: Scaled up replica set ensemble-graph-510ff-85d6b4bb57 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: ensemble-graph-510ff.18a9597c1b040769 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: f1223762-61cc-4018-a04b-343aa8ba821f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-510ff namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: c4189e80-b286-42c0-9563-fd960343b5d0 kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: InferenceGraph [ensemble-graph-510ff] is Ready metadata: creationTimestamp: "2026-04-24T16:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:37Z" name: ensemble-graph-510ff.18a9597c1faee719 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 8c5f9614-7afc-4e1f-8e34-b5b0ab479560 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-510ff namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: ca0d4a73-ec74-4422-a4b1-7ae89454d5ad kind: Event lastTimestamp: "2026-04-24T16:59:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:07Z" name: ensemble-graph-510ff.18a9597f9972bec9 namespace: kserve-ci-e2e-test resourceVersion: "20140" uid: 2a8bc2ac-0fe0-46c9-b3a8-349f29c8171a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-510ff namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: ca0d4a73-ec74-4422-a4b1-7ae89454d5ad kind: Event lastTimestamp: "2026-04-24T16:59:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:07Z" name: ensemble-graph-510ff.18a9597f99c0ba14 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: fa7f7a2a-9e6b-4202-bcbf-a6cc4a9781b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:57:27.295325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18890" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-f6171-6bffd6cbc4-82phr to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596bc033570b namespace: kserve-ci-e2e-test resourceVersion: "18894" uid: be586ffd-c909-44c6-95c3-c22e3acb3020 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18898" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596bd931ad17 namespace: kserve-ci-e2e-test resourceVersion: "18914" uid: 022a2cfd-da3c-4aae-b0f9-0ea27e763e19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6171} kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596bda8d9efb namespace: kserve-ci-e2e-test resourceVersion: "18916" uid: 8bac8e60-092f-469d-8faf-7b075cd7fff3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6171} kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'Created container: ensemble-graph-f6171' metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596bdd88bc05 namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: f529dddc-ed89-4afd-88d2-032edf089579 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6171} kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: Started container ensemble-graph-f6171 metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596bddfe8c33 namespace: kserve-ci-e2e-test resourceVersion: "18918" uid: bc78db1d-e3b1-46ce-b4ce-8da51f16f710 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6171} kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Stopping container ensemble-graph-f6171 metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596e170e0321 namespace: kserve-ci-e2e-test resourceVersion: "19020" uid: b0848174-8241-41b7-ae7e-803cb0c62859 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6171} kind: Pod name: ensemble-graph-f6171-6bffd6cbc4-82phr namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 2b064f2e-90a8-488f-98e1-8ae698ce7f7d kind: Event lastTimestamp: "2026-04-24T16:58:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:04Z" name: ensemble-graph-f6171-6bffd6cbc4-82phr.18a9596e9f475c19 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 36da1bf3-0d91-4782-8e7a-c0b10ba04749 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-f6171-6bffd6cbc4 namespace: kserve-ci-e2e-test resourceVersion: "18886" uid: db04828e-b6de-4f93-b1b5-a31755f36ea4 kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'Created pod: ensemble-graph-f6171-6bffd6cbc4-82phr' metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171-6bffd6cbc4.18a9596bbfc700b0 namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: a0c8150f-bb67-490b-9c58-b152dbf36659 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18884" uid: e4b1d39f-5036-48b7-9ea9-c9395be99229 kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: Scaled up replica set ensemble-graph-f6171-6bffd6cbc4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171.18a9596bbec67fec namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 0ac1421b-a598-4118-9be4-35791e87fe02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: f19ad67c-e9c1-46b1-b6d3-f9288c31578d kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: InferenceGraph [ensemble-graph-f6171] is Ready metadata: creationTimestamp: "2026-04-24T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:27Z" name: ensemble-graph-f6171.18a9596bc5e290dd namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 60e7ae4e-8f06-43d5-af80-5a57d9003ebc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:47:12.240013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dc8c0c133a namespace: kserve-ci-e2e-test resourceVersion: "13810" uid: 0c096789-b3dc-4998-b9be-ed35fc025823 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2c4dd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dc99f14021 namespace: kserve-ci-e2e-test resourceVersion: "13823" uid: 187a4ffe-dace-413d-9524-df10361a1a8a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13819" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dcc950ab5c namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: 336b61a4-131c-424b-a7cb-de3a36c17137 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dccabd5985 namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: da394735-b30f-41d5-9916-d28d901b1b7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dccdd0408a namespace: kserve-ci-e2e-test resourceVersion: "13853" uid: 6a7e23fc-0e88-41a9-9883-5e780902240c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dcce3b9b27 namespace: kserve-ci-e2e-test resourceVersion: "13855" uid: 1e69fe34-b63a-4645-86f1-014d1878f937 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dcce46d0a1 namespace: kserve-ci-e2e-test resourceVersion: "13858" uid: 770445e0-5fb3-4914-9367-ab7dfd2e624d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dcd1f7bcfa namespace: kserve-ci-e2e-test resourceVersion: "13865" uid: 66eefe31-4e42-42c3-8622-bb7332372cf6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dcd26d0c9d namespace: kserve-ci-e2e-test resourceVersion: "13870" uid: ef4fe29c-38e2-4bb1-9fd4-8b0bea77d065 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:47:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:51Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a958dd455cfad8 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 38056fe7-ee1c-499e-8cc7-792cd804f2d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:27Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a9595de36cb471 namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: 7c0e9e88-49f1-44f0-bb3b-39fba6129136 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: d22fadcc-27c7-47ce-8cd4-29c8fda89621 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:27Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt.18a9595de36d0f1e namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: fc494dc9-9fb4-4d8b-b619-d3cd04afc471 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2c4dd-predictor-bdc7b5877 namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 69b0faad-b0de-48a9-b927-7a6879995506 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Created pod: error-404-isvc-2c4dd-predictor-bdc7b5877-vpdlt' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd-predictor-bdc7b5877.18a958dc8ba4367d namespace: kserve-ci-e2e-test resourceVersion: "13808" uid: a2ea2476-6e7f-4d66-88fb-65641f7e5c55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: 9b143b90-efe9-4470-8f8c-9a95c1207db9 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Scaled up replica set error-404-isvc-2c4dd-predictor-bdc7b5877 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd-predictor.18a958dc89de33e0 namespace: kserve-ci-e2e-test resourceVersion: "13801" uid: 9612e2ab-6aef-4e81-acf6-350292c60169 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: af4f6da6-03b8-4b6f-bb91-c55e5b0873db kind: Event lastTimestamp: "2026-04-24T16:47:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:57Z" name: error-404-isvc-2c4dd-predictor.18a958e008d98bca namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 2aa09297-d346-42a7-815e-877a3cece34d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: af4f6da6-03b8-4b6f-bb91-c55e5b0873db kind: Event lastTimestamp: "2026-04-24T16:47:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:57Z" name: error-404-isvc-2c4dd-predictor.18a958e00932150b namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: ff9e950d-0b7d-4608-89c1-cc8f8c4fe999 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: af4f6da6-03b8-4b6f-bb91-c55e5b0873db kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:12Z" name: error-404-isvc-2c4dd-predictor.18a958ea8673ab48 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 5d7a4160-97da-4407-9b6f-fd2fe3c24c7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: af4f6da6-03b8-4b6f-bb91-c55e5b0873db kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:12Z" name: error-404-isvc-2c4dd-predictor.18a958ea874e0dcd namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 1d0523b0-a507-4ec3-981c-0742388264fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 4b5ba39e-0cb2-4127-bdb3-19a574e9cef3 kind: Event lastTimestamp: "2026-04-24T16:48:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:01Z" name: error-404-isvc-2c4dd.18a958dc89b9edf1 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 394d7fdf-f63d-4dc4-95ff-d2be46fc6581 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 4b5ba39e-0cb2-4127-bdb3-19a574e9cef3 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-2c4dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2c4dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd.18a958dc8a0eee59 namespace: kserve-ci-e2e-test resourceVersion: "13803" uid: 7b0757f6-9734-4f19-8b8a-afa47169d5f1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 4b5ba39e-0cb2-4127-bdb3-19a574e9cef3 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2c4dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-2c4dd.18a958dc8a0f4074 namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: 6efde5a3-c077-4fc0-b161-eb3f93b441ed reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 4b5ba39e-0cb2-4127-bdb3-19a574e9cef3 kind: Event lastTimestamp: "2026-04-24T16:48:01Z" message: InferenceService [error-404-isvc-2c4dd] is Ready metadata: creationTimestamp: "2026-04-24T16:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:01Z" name: error-404-isvc-2c4dd.18a958e7fff73b3b namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 737a3425-f600-4cba-bd84-a8fcc358f73c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:47:48.111260Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-30dd6-predictor-79f6568b65-kq94r to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e4e623ddcc namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: d69187de-1734-4642-b65c-1ab5d3e1233b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-30dd6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e4ee1cd4ee namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 7740caa1-0c33-49ac-b8b0-83e14315a6df reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e5236f20e3 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 2ccc868e-ea5e-41ae-936f-b8da8634e0c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e525dcab43 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 9e4a8d4d-21e1-4f8b-b55a-3d41230e6ac3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e529430ac5 namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 3e4daecc-fb5c-4ab7-a3e6-d0ceb413178b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e529ac93ad namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: db9b6e2e-21ab-46a9-9f23-6fe1881159e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e529b73cfa namespace: kserve-ci-e2e-test resourceVersion: "14369" uid: 2ebc7161-7278-4473-9568-6320d996892c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e52d0f21ec namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 89ca7acf-7d3c-47eb-9151-e767190f2000 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e52d7e667a namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: c16d6878-940f-47fa-9e17-4757f8388de7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:57:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:06Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a958e5724aa532 namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: d2e558cf-970c-487e-90c9-3481ad6610f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a959661ce8475c namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: 4d0a289f-e2bd-45ed-9a0a-17cf933ce66b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a959661ce8ee91 namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 7ea02a59-9b63-46b1-a69b-e152580b5363 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: a8a1736d-f782-4dea-9285-96f0b6062300 kind: Event lastTimestamp: "2026-04-24T16:57:06Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:06Z" name: error-404-isvc-30dd6-predictor-79f6568b65-kq94r.18a95966e6a1344e namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: a920a945-7a95-4bf5-b6c4-52e66efa818a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-30dd6-predictor-79f6568b65 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: fbbe4fb8-bd35-40cd-a0f5-5d1773f370c6 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'Created pod: error-404-isvc-30dd6-predictor-79f6568b65-kq94r' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6-predictor-79f6568b65.18a958e4e5bcbb86 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 0b008ab6-0583-441a-bfc4-dc49e6c79cee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 6d7b8593-8dc0-4fbb-a8a2-5bf0bc7a5f40 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: Scaled up replica set error-404-isvc-30dd6-predictor-79f6568b65 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6-predictor.18a958e4e3e27da9 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 45b5ad98-30bf-48ff-9126-6f7244ceba54 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: d1e2d318-fdd5-4002-af97-21593a8f6d75 kind: Event lastTimestamp: "2026-04-24T16:48:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:03Z" name: error-404-isvc-30dd6-predictor.18a958e8625055d8 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 89523f12-aff1-42fb-886b-449e59bdf68e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: d1e2d318-fdd5-4002-af97-21593a8f6d75 kind: Event lastTimestamp: "2026-04-24T16:48:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:03Z" name: error-404-isvc-30dd6-predictor.18a958e862c7c240 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: d3b2cf7f-9653-42a5-bffc-002db3a7154f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: d1e2d318-fdd5-4002-af97-21593a8f6d75 kind: Event lastTimestamp: "2026-04-24T16:49:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:03Z" name: error-404-isvc-30dd6-predictor.18a958ebe17a4e5f namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 07c370b8-4c4a-49a1-a547-342fea82b46c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: d1e2d318-fdd5-4002-af97-21593a8f6d75 kind: Event lastTimestamp: "2026-04-24T16:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:03Z" name: error-404-isvc-30dd6-predictor.18a958ebe1ed668a namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 323fc2ff-96aa-4c67-be47-23d22ab18fa9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: a6f49c5d-1cf2-439d-8bc2-686088ab9838 kind: Event lastTimestamp: "2026-04-24T16:48:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:36Z" name: error-404-isvc-30dd6.18a958e4e3a170c6 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: fbef2d0c-a45f-49db-a3cb-db13e7ac4682 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: a6f49c5d-1cf2-439d-8bc2-686088ab9838 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-30dd6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-30dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6.18a958e4e495e6cb namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: 9c2fd8c2-2dcc-4fc0-8dd0-f81468a67b94 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: a6f49c5d-1cf2-439d-8bc2-686088ab9838 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-30dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:48Z" name: error-404-isvc-30dd6.18a958e4e4964bef namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: 4d308124-403f-4fbd-b773-fa141302802f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: a6f49c5d-1cf2-439d-8bc2-686088ab9838 kind: Event lastTimestamp: "2026-04-24T16:48:36Z" message: InferenceService [error-404-isvc-30dd6] is Ready metadata: creationTimestamp: "2026-04-24T16:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:36Z" name: error-404-isvc-30dd6.18a958f02d6c042b namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: acac2c5a-4d41-4b05-8602-c2e22b8a5a89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:45:37.986987Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12641" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:45:37Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958c69a2108e9 namespace: kserve-ci-e2e-test resourceVersion: "12646" uid: 257fa933-a74a-42d0-a628-c008b89d1759 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-32209-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:38Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958c6a61dab1c namespace: kserve-ci-e2e-test resourceVersion: "12670" uid: 3814f497-e668-4a50-8076-52e4b052a933 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12659" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:45:39Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958c6da4c742c namespace: kserve-ci-e2e-test resourceVersion: "12724" uid: 82ff4f01-e9ee-41e7-9fe0-707ca92deb48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:39Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958c6db9ffcbe namespace: kserve-ci-e2e-test resourceVersion: "12728" uid: fbe9e555-112b-4e01-8bab-232e28154575 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" in 15.262s (15.262s including waiting). Image size: 1334780514 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958ca695ee19d namespace: kserve-ci-e2e-test resourceVersion: "12911" uid: 094c5459-7c8e-44ac-a4c2-93e702096c2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958ca6d079be9 namespace: kserve-ci-e2e-test resourceVersion: "12914" uid: 2933f200-7333-4b14-84cc-5c49ac79425d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958ca6d83a1ca namespace: kserve-ci-e2e-test resourceVersion: "12917" uid: bff00035-3add-4557-a47b-70f75de3c32c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958ca6d8d480a namespace: kserve-ci-e2e-test resourceVersion: "12918" uid: 2cee3d76-a2e7-4ab7-8f54-21ae32b021fa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.531s (2.531s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:56Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958cb046ccfe8 namespace: kserve-ci-e2e-test resourceVersion: "12938" uid: affa3430-1adf-452c-98e6-d9305109fca0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:57Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958cb08b63b24 namespace: kserve-ci-e2e-test resourceVersion: "12940" uid: 0fc5a831-543d-44bd-b793-7c6761aaee1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:45:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:57Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958cb0930ab5f namespace: kserve-ci-e2e-test resourceVersion: "12943" uid: e8002972-33ce-4b22-9ade-2fdc0a769f9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958cb0d28b2f8 namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: 0de51fa1-e178-4f56-99c1-5ea8a99be6c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958dc8686b8c3 namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: fe783f79-b272-43bd-ba6c-0cee9c4d57ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958dc8687320c namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: d21755aa-370f-4bec-8207-0b51c9f7b062 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5 namespace: kserve-ci-e2e-test resourceVersion: "12643" uid: 3a5f72e4-d8de-42d7-9078-2fa28a50203d kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5.18a958dcbf539122 namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: 4422a50b-f9a9-4459-8984-2030154eb9ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32209-predictor-9c8dc8f6c namespace: kserve-ci-e2e-test resourceVersion: "12635" uid: fca20f48-3fb4-48c0-ae7b-e04fd29d08ce kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'Created pod: error-404-isvc-32209-predictor-9c8dc8f6c-5r4r5' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: error-404-isvc-32209-predictor-9c8dc8f6c.18a958c699c22a72 namespace: kserve-ci-e2e-test resourceVersion: "12644" uid: 946e2e33-a79a-4179-b467-6c450e3fe2fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12632" uid: b8296d45-9f89-4fe4-a0f1-9a4178bf2a52 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: Scaled up replica set error-404-isvc-32209-predictor-9c8dc8f6c from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: error-404-isvc-32209-predictor.18a958c6980248f0 namespace: kserve-ci-e2e-test resourceVersion: "12637" uid: 4b4d4e72-c829-4438-bcf4-3dc5d4ea201e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12634" uid: 948c9513-6c7a-4eef-bb75-8b423a836ae1 kind: Event lastTimestamp: "2026-04-24T16:46:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:23Z" name: error-404-isvc-32209-predictor.18a958ca175714b4 namespace: kserve-ci-e2e-test resourceVersion: "13249" uid: 7074f04b-3f67-446e-b58b-ccf1dfca0ef5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12634" uid: 948c9513-6c7a-4eef-bb75-8b423a836ae1 kind: Event lastTimestamp: "2026-04-24T16:46:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:23Z" name: error-404-isvc-32209-predictor.18a958ca180ca9a7 namespace: kserve-ci-e2e-test resourceVersion: "13250" uid: ccf1708f-e9a2-436f-8325-b8f4385d082c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 948c9513-6c7a-4eef-bb75-8b423a836ae1 kind: Event lastTimestamp: "2026-04-24T16:47:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:08Z" name: error-404-isvc-32209-predictor.18a958d4952fb3b5 namespace: kserve-ci-e2e-test resourceVersion: "13680" uid: e790a190-3358-444a-b6e9-2296bd1306e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 948c9513-6c7a-4eef-bb75-8b423a836ae1 kind: Event lastTimestamp: "2026-04-24T16:47:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:08Z" name: error-404-isvc-32209-predictor.18a958d495b3daeb namespace: kserve-ci-e2e-test resourceVersion: "13681" uid: 23943c11-9de0-4e98-b86b-8d3416ba5783 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "12617" uid: 66bf6f67-4dc2-4d0f-8ae1-3350c21c5457 kind: Event lastTimestamp: "2026-04-24T16:46:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:46:43Z" name: error-404-isvc-32209.18a958c697daa126 namespace: kserve-ci-e2e-test resourceVersion: "13434" uid: 66c30518-a017-4185-a6da-094df3e0b445 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "13430" uid: 66bf6f67-4dc2-4d0f-8ae1-3350c21c5457 kind: Event lastTimestamp: "2026-04-24T16:46:43Z" message: InferenceService [error-404-isvc-32209] is Ready metadata: creationTimestamp: "2026-04-24T16:46:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:46:43Z" name: error-404-isvc-32209.18a958d5ce835b3e namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: 6c0e39c1-ca97-4d54-9e8e-b452d297a099 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:06:52.318068Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23448" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3827a-predictor-69b76c5cb9-hq259 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef4e2e2fbb namespace: kserve-ci-e2e-test resourceVersion: "23452" uid: e8052a41-05fa-4973-afbd-2f0618b8ad99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23458" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef67ce6b5a namespace: kserve-ci-e2e-test resourceVersion: "23470" uid: 97c083df-9474-43cf-aeae-27012e3c9f4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef6936c17b namespace: kserve-ci-e2e-test resourceVersion: "23474" uid: 2ebc4200-472f-4d52-8d23-5083bace8331 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef6c0f20f3 namespace: kserve-ci-e2e-test resourceVersion: "23478" uid: 5385da4d-defa-4256-bb0e-872fcd3d8362 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef6c783270 namespace: kserve-ci-e2e-test resourceVersion: "23479" uid: 2388dd55-ba79-41cf-ab6e-f51715d64d2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef6c8265ac namespace: kserve-ci-e2e-test resourceVersion: "23480" uid: 64dca20c-f94f-4c69-b3cc-3cc7f07448b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef705d8415 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: 39e11226-d66f-4d26-a58e-cec851f6279d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ef70c7ba06 namespace: kserve-ci-e2e-test resourceVersion: "23483" uid: 8a617a22-9b8a-40bd-8fa3-204f9acebd02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:07:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:30Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959efb56d2051 namespace: kserve-ci-e2e-test resourceVersion: "23922" uid: e611961a-240c-45ac-884b-b448c0883846 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ffa7a8a4d0 namespace: kserve-ci-e2e-test resourceVersion: "24328" uid: b6746d90-8669-420a-ab15-62a21a2e9412 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a959ffa7a9cfe4 namespace: kserve-ci-e2e-test resourceVersion: "24330" uid: 819d7176-c1fd-4b38-96ae-81913f027db4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259 namespace: kserve-ci-e2e-test resourceVersion: "23450" uid: ded5d261-7d83-4e14-bcb0-56c408e51992 kind: Event lastTimestamp: "2026-04-24T17:08:05Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:05Z" name: error-404-isvc-3827a-predictor-69b76c5cb9-hq259.18a95a003dc55d7d namespace: kserve-ci-e2e-test resourceVersion: "24407" uid: 493a99d7-9206-49cc-b91c-0e8d28145d48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3827a-predictor-69b76c5cb9 namespace: kserve-ci-e2e-test resourceVersion: "23440" uid: 2cc98614-07f3-4550-9188-c6a707409102 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created pod: error-404-isvc-3827a-predictor-69b76c5cb9-hq259' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor-69b76c5cb9.18a959ef4dbd2266 namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 29e84e8c-e11d-4b09-9f8e-59ce4e58ba8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23436" uid: 56d4a4f5-6bf2-4eb6-8fa0-ceee5dd7d8e8 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Scaled up replica set error-404-isvc-3827a-predictor-69b76c5cb9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a-predictor.18a959ef4c1c52f0 namespace: kserve-ci-e2e-test resourceVersion: "23442" uid: 1397a508-b94f-436c-a126-b484800f9035 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23439" uid: 4b9da850-ae63-4a3f-97a8-7d162c8b7e72 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: error-404-isvc-3827a-predictor.18a959f2cabd7e89 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: 8404e3b9-64fe-468b-affb-9ca4e3e1551e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23439" uid: 4b9da850-ae63-4a3f-97a8-7d162c8b7e72 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: error-404-isvc-3827a-predictor.18a959f2cb25cbb9 namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: de6aab23-c95d-4782-8548-a146df549105 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23613" uid: 4b9da850-ae63-4a3f-97a8-7d162c8b7e72 kind: Event lastTimestamp: "2026-04-24T17:07:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:22Z" name: error-404-isvc-3827a-predictor.18a959f649d3aa47 namespace: kserve-ci-e2e-test resourceVersion: "23726" uid: c36869df-2609-4052-9ed5-58db388b06b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23613" uid: 4b9da850-ae63-4a3f-97a8-7d162c8b7e72 kind: Event lastTimestamp: "2026-04-24T17:07:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:22Z" name: error-404-isvc-3827a-predictor.18a959f64a1e07ec namespace: kserve-ci-e2e-test resourceVersion: "23728" uid: 5cf0aa4a-fddc-4d5d-84ca-b802343c438f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23407" uid: 518ba5ff-3cda-4c5e-a6ef-e7b0c3acab5c kind: Event lastTimestamp: "2026-04-24T17:07:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:40Z" name: error-404-isvc-3827a.18a959ef4c1fb13f namespace: kserve-ci-e2e-test resourceVersion: "24033" uid: ea7213aa-62a0-4caf-a6c6-2a089acb427e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23407" uid: 518ba5ff-3cda-4c5e-a6ef-e7b0c3acab5c kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-3827a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3827a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a.18a959ef4ca9de9a namespace: kserve-ci-e2e-test resourceVersion: "23445" uid: 1c031a13-d828-49ed-b705-61c05a49a25b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23407" uid: 518ba5ff-3cda-4c5e-a6ef-e7b0c3acab5c kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3827a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-3827a.18a959ef4caa20bc namespace: kserve-ci-e2e-test resourceVersion: "23446" uid: 110cd5ef-7f7b-4852-b70f-a01b9d12facc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "24031" uid: 518ba5ff-3cda-4c5e-a6ef-e7b0c3acab5c kind: Event lastTimestamp: "2026-04-24T17:07:40Z" message: InferenceService [error-404-isvc-3827a] is Ready metadata: creationTimestamp: "2026-04-24T17:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:40Z" name: error-404-isvc-3827a.18a959fa705fd1ee namespace: kserve-ci-e2e-test resourceVersion: "24032" uid: 119e132e-34dc-481f-8398-92ecb50331e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:57:03.130910Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18607" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3eac5-predictor-5466454db9-flmsv to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959661fe3a023 namespace: kserve-ci-e2e-test resourceVersion: "18611" uid: 573b3cde-92a5-4379-be6f-ddd73c82e88e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3eac5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959662db0a998 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: 6f37aa78-9174-461d-bbe0-a61726189001 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959665cd43d54 namespace: kserve-ci-e2e-test resourceVersion: "18639" uid: fd067fbb-b133-4e61-880e-24517509a00c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959665e4f09cd namespace: kserve-ci-e2e-test resourceVersion: "18641" uid: fff38da6-9c33-406f-aeb8-e77c9d5f1adf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a95966615219d2 namespace: kserve-ci-e2e-test resourceVersion: "18643" uid: 39e9232a-cc44-422f-9ee1-8591fef475d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a9596661bcb066 namespace: kserve-ci-e2e-test resourceVersion: "18644" uid: 0d71e30f-bbf6-414a-a0b3-42b419db7273 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a9596661c87662 namespace: kserve-ci-e2e-test resourceVersion: "18645" uid: 1f267ee0-dabb-44ed-a247-27d8485bfdcf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a95966649981f9 namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: b1a4fed5-6f11-4174-8c22-bb4f70b8c9a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a9596665033f93 namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: b9e9ff70-a5f5-4625-8e7e-03a2e0963bcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:57:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:42Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a95966d6e5d05c namespace: kserve-ci-e2e-test resourceVersion: "19180" uid: 183a1587-8b10-428d-a6dd-f90ee2ad6bbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959766df22c6a namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 2efda9eb-23e9-4d12-98cb-da2829d28d4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3eac5-predictor-5466454db9-flmsv namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 92839f22-2bdc-4478-8460-1f9edbf864e9 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-3eac5-predictor-5466454db9-flmsv.18a959766df2911f namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 05b9aaa8-6d41-481a-82e7-f667b1ffe1e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3eac5-predictor-5466454db9 namespace: kserve-ci-e2e-test resourceVersion: "18599" uid: 79ca645f-eb78-43a5-a996-5a02c568fa1d kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: 'Created pod: error-404-isvc-3eac5-predictor-5466454db9-flmsv' metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-3eac5-predictor-5466454db9.18a959661f09f71a namespace: kserve-ci-e2e-test resourceVersion: "18608" uid: 9768ad68-c7ff-4b60-aad1-1a221e27a1be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18596" uid: 1b57e95e-1304-417e-848e-1d1ee1d3dcb0 kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: Scaled up replica set error-404-isvc-3eac5-predictor-5466454db9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:03Z" name: error-404-isvc-3eac5-predictor.18a959661d0d77b5 namespace: kserve-ci-e2e-test resourceVersion: "18602" uid: 2e2c8741-7411-481c-9a1a-8cca81969f37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18598" uid: 21839441-3c05-48aa-b914-ddf8d7741ea4 kind: Event lastTimestamp: "2026-04-24T16:57:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:48Z" name: error-404-isvc-3eac5-predictor.18a959699b922250 namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: f340c172-38dc-45a7-b6eb-8cdab354c869 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18598" uid: 21839441-3c05-48aa-b914-ddf8d7741ea4 kind: Event lastTimestamp: "2026-04-24T16:57:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:48Z" name: error-404-isvc-3eac5-predictor.18a959699c064a19 namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: b86c6fd1-47ff-419e-b54a-7fa37d4c264d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: 21839441-3c05-48aa-b914-ddf8d7741ea4 kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:03Z" name: error-404-isvc-3eac5-predictor.18a9596d1ac81e85 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 1d442e12-2329-49a7-9d05-941d0055991d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: 21839441-3c05-48aa-b914-ddf8d7741ea4 kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:03Z" name: error-404-isvc-3eac5-predictor.18a9596d1b3d55a2 namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 8a899409-d743-4a7b-90f9-919592ea41d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "18567" uid: f727c63d-20e1-4068-816d-8d367c546ec5 kind: Event lastTimestamp: "2026-04-24T16:57:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:52Z" name: error-404-isvc-3eac5.18a959661cd982ee namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: 781b95bc-c92f-443c-82f8-80b65804c193 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: f727c63d-20e1-4068-816d-8d367c546ec5 kind: Event lastTimestamp: "2026-04-24T16:57:52Z" message: InferenceService [error-404-isvc-3eac5] is Ready metadata: creationTimestamp: "2026-04-24T16:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:52Z" name: error-404-isvc-3eac5.18a9597192a396a4 namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: bb1489f0-cdf8-44c9-b7b5-8c4f51f427bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:57:37.663809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-510ff-predictor-7b675fd9f5-vq786 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e2a35dc59 namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: 05f1dc5b-0017-4331-a11a-c1a720e899bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19111" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e43f2e87f namespace: kserve-ci-e2e-test resourceVersion: "19125" uid: 46ca66cc-eb49-439e-b36d-462a88ffd378 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e454fc574 namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: 9374207c-6312-4ecf-8a32-551313f6c09d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e483a2ad0 namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: 8c531ba9-3c66-4f25-8d02-d000b27f46c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e48c89ba6 namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: 71fc2c3f-8765-4c46-848c-75a679f6643f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e48d4d128 namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: cfb0e1eb-6b0d-41d4-94af-8a52b6a58343 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e4ba3b311 namespace: kserve-ci-e2e-test resourceVersion: "19131" uid: 056dd79e-7502-440b-aa89-09bf722ff89b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e4c0b1e0f namespace: kserve-ci-e2e-test resourceVersion: "19132" uid: ba1ae7b3-3271-4f32-82ed-d55153e3ca0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T17:06:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:55Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a9596e8d63b840 namespace: kserve-ci-e2e-test resourceVersion: "23517" uid: d58fc2e4-a9a4-4af7-b449-5b1bac7842d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a959ef49f7fa1a namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: 0fd33ebc-a908-406b-a705-94fa26f842e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a959ef49f884ba namespace: kserve-ci-e2e-test resourceVersion: "23431" uid: 304809f4-f073-4b3e-a01e-cfa0d744b6e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 28719115-6329-46e4-8521-9fbf569b1aca kind: Event lastTimestamp: "2026-04-24T17:06:55Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:55Z" name: error-404-isvc-510ff-predictor-7b675fd9f5-vq786.18a959f001c2f170 namespace: kserve-ci-e2e-test resourceVersion: "23516" uid: 80b70026-df9a-4942-aeb6-9eda5907b19f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-510ff-predictor-7b675fd9f5 namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: e50116ad-7ea8-428f-a5ee-b2d0dc5613d2 kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'Created pod: error-404-isvc-510ff-predictor-7b675fd9f5-vq786' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-510ff-predictor-7b675fd9f5.18a9596e29bcf241 namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: 30144c15-5fd6-451b-875d-c3943f513781 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: e252904c-4b26-4142-987b-c322d345c16d kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Scaled up replica set error-404-isvc-510ff-predictor-7b675fd9f5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-510ff-predictor.18a9596e273c9c26 namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: ff583ee4-83de-4326-9f00-bb7feffc2ebf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: b3519c3e-2f34-448a-ba2a-0324c4adf0ad kind: Event lastTimestamp: "2026-04-24T16:57:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:52Z" name: error-404-isvc-510ff-predictor.18a95971a5b99a76 namespace: kserve-ci-e2e-test resourceVersion: "19287" uid: 28ea34c8-5930-4358-9c6f-563151b5dba5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: b3519c3e-2f34-448a-ba2a-0324c4adf0ad kind: Event lastTimestamp: "2026-04-24T16:57:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:52Z" name: error-404-isvc-510ff-predictor.18a95971a6086f57 namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 582e1c18-13dd-4f72-9570-a5533e5897e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: b3519c3e-2f34-448a-ba2a-0324c4adf0ad kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: error-404-isvc-510ff-predictor.18a9597524d3d96d namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: f7f8d307-d13a-4270-a200-959e16c7af48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: b3519c3e-2f34-448a-ba2a-0324c4adf0ad kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: error-404-isvc-510ff-predictor.18a959752543b43b namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: ef503c29-94c7-4d80-98a8-5c60e9fda8cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 7898d65d-4861-4997-8eb6-45f6ad5f1fbc kind: Event lastTimestamp: "2026-04-24T16:58:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:25Z" name: error-404-isvc-510ff.18a9596e270e8f04 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: eb90934b-2224-4df4-b3c3-d0c06e60cd71 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 7898d65d-4861-4997-8eb6-45f6ad5f1fbc kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-510ff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-510ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-510ff.18a9596e27b3b6d1 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: 133b3c7b-b0cc-42f9-87c8-0f63925ac8e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 7898d65d-4861-4997-8eb6-45f6ad5f1fbc kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-510ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-510ff.18a9596e27b41da9 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 31e8b080-2261-4d4a-9b72-d227ef99f4f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 7898d65d-4861-4997-8eb6-45f6ad5f1fbc kind: Event lastTimestamp: "2026-04-24T16:58:25Z" message: InferenceService [error-404-isvc-510ff] is Ready metadata: creationTimestamp: "2026-04-24T16:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:25Z" name: error-404-isvc-510ff.18a959794b402707 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 39c2710c-0e9c-4e46-af89-3bfe07c4d79c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:58:13.394475Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959767bec9151 namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 7734dbf7-2d54-4682-a70d-0df39d70c083 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a9597695bb66a9 namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 49bcaca9-88a3-4bb6-8e5e-1222dee67949 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a95976971bbcb9 namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: c15f4bd8-c264-4a36-bee3-75d2a5b97955 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a9597699c7237f namespace: kserve-ci-e2e-test resourceVersion: "19616" uid: 6c2ab998-80c6-4ae7-982d-711b2db7ec73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959769a33ce7f namespace: kserve-ci-e2e-test resourceVersion: "19618" uid: cb11e837-7817-416f-8dac-6ef5bbcc08f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959769a3e6099 namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: 677c2457-eaff-4417-ac3e-0490423af530 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959769d2205de namespace: kserve-ci-e2e-test resourceVersion: "19620" uid: b86a8e5d-bf69-442d-89b5-1b6c5ba32b80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959769d8af63b namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: 5981f1b3-bd78-4bc4-9487-bbc256ea03a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T17:07:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:30Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a95976baf0ae8d namespace: kserve-ci-e2e-test resourceVersion: "23928" uid: d61a1501-7ce5-4307-8405-f6d6e41100ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959f7a2a51c50 namespace: kserve-ci-e2e-test resourceVersion: "23848" uid: fa64c77e-9d84-4401-a6ca-5be0e4865fb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959f7a2a6409e namespace: kserve-ci-e2e-test resourceVersion: "23850" uid: 68c2c227-dbca-48af-9884-87bdaf52e614 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 5735bf3c-3cfb-4792-825c-8d01882932ba kind: Event lastTimestamp: "2026-04-24T17:07:30Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:30Z" name: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8.18a959f82f5b4af3 namespace: kserve-ci-e2e-test resourceVersion: "23927" uid: 33614a01-7cdb-4262-bc60-b68adcf9dc35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-77cb4-predictor-6cd577bd49 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: 48d42e51-8ce8-4e4a-8b7e-5b70bf10c516 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created pod: error-404-isvc-77cb4-predictor-6cd577bd49-pnql8' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor-6cd577bd49.18a959767af91859 namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: bfeffb14-8197-4163-9057-5acc2c8a1068 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: 53d315a8-7159-4c95-81bf-142fe268f36a kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Scaled up replica set error-404-isvc-77cb4-predictor-6cd577bd49 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4-predictor.18a9597677c2648c namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: d90aa918-133e-405b-9e87-7aa281c9a142 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 0857a6b3-a4e1-4d6d-8cc8-29cb2020d12c kind: Event lastTimestamp: "2026-04-24T16:58:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:28Z" name: error-404-isvc-77cb4-predictor.18a95979f658aea1 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: de0ea2f6-ef27-418f-ba92-9dee62c06a95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 0857a6b3-a4e1-4d6d-8cc8-29cb2020d12c kind: Event lastTimestamp: "2026-04-24T16:58:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:28Z" name: error-404-isvc-77cb4-predictor.18a95979f6c94a7b namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 00a1b90c-7624-4ec1-9148-32c55a37e5dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 0857a6b3-a4e1-4d6d-8cc8-29cb2020d12c kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: error-404-isvc-77cb4-predictor.18a9597d758d1399 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 107fbb61-af02-4f64-93d2-2fecd33f2179 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 0857a6b3-a4e1-4d6d-8cc8-29cb2020d12c kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: error-404-isvc-77cb4-predictor.18a9597d76021377 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: df1092e1-59be-4a46-b120-b788bb1847a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 3daea503-b560-4b92-be0f-abaf6ae5309a kind: Event lastTimestamp: "2026-04-24T16:59:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:59:00Z" name: error-404-isvc-77cb4.18a95976778ca6eb namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: b86483eb-0292-4a55-aa0a-ae45b82259ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 3daea503-b560-4b92-be0f-abaf6ae5309a kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-77cb4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-77cb4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4.18a9597677fa8886 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 1be448b7-19f5-4e87-9cf6-83936d27a0ce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 3daea503-b560-4b92-be0f-abaf6ae5309a kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-77cb4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:13Z" name: error-404-isvc-77cb4.18a9597677fae1f3 namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: ec95b631-3f63-403f-96b5-25f96085576b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 3daea503-b560-4b92-be0f-abaf6ae5309a kind: Event lastTimestamp: "2026-04-24T16:59:00Z" message: InferenceService [error-404-isvc-77cb4] is Ready metadata: creationTimestamp: "2026-04-24T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:59:00Z" name: error-404-isvc-77cb4.18a95981764daf45 namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 2b5c990d-f820-403b-87ca-df07d4446490 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:07:28.175988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23865" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8edb5-predictor-667c7d688c-fwxdj to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7a77a91a5 namespace: kserve-ci-e2e-test resourceVersion: "23869" uid: fac542da-e68c-4d47-a201-f84144199e7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23875" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c105eb67 namespace: kserve-ci-e2e-test resourceVersion: "23893" uid: 63a8590c-efe5-4de4-9251-0d9606e4d8c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c2470108 namespace: kserve-ci-e2e-test resourceVersion: "23895" uid: 558a2e9f-9e8c-4a47-af85-82c061bc19c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c45999f9 namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: c2628896-62e6-409d-bda7-ed18a2e94e6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c4cd68b5 namespace: kserve-ci-e2e-test resourceVersion: "23899" uid: 18d9d586-748c-4fb3-b815-5a4318723638 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c4e18d9a namespace: kserve-ci-e2e-test resourceVersion: "23900" uid: d58f7168-596b-4bb2-9aa3-1e675720ec7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c7954de8 namespace: kserve-ci-e2e-test resourceVersion: "23901" uid: 25632800-8479-49e3-8df5-91d0c8089df3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f7c814b5f8 namespace: kserve-ci-e2e-test resourceVersion: "23902" uid: c55e8e38-197d-4ce8-876d-c5501a8c162f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:08:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:06Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a959f81f360244 namespace: kserve-ci-e2e-test resourceVersion: "24418" uid: 2dddb256-cec8-4946-8b36-babed5c07c4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:24:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:24:47Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a95ae99f766111 namespace: kserve-ci-e2e-test resourceVersion: "31895" uid: cd4a0d00-7e73-49d7-acc4-d6a04c37feae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 81afd582-5ad6-4837-93c1-20edce9f24c9 kind: Event lastTimestamp: "2026-04-24T17:24:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:24:47Z" name: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj.18a95ae99f77c646 namespace: kserve-ci-e2e-test resourceVersion: "31896" uid: 132e0f59-5051-4d1b-9f4d-b8de4a0404f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8edb5-predictor-667c7d688c namespace: kserve-ci-e2e-test resourceVersion: "23858" uid: ae0dbca5-4296-4ea5-a64b-c5792ad8b08b kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created pod: error-404-isvc-8edb5-predictor-667c7d688c-fwxdj' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor-667c7d688c.18a959f7a6e5ade5 namespace: kserve-ci-e2e-test resourceVersion: "23866" uid: baa8ba78-f8e0-4873-9598-519b88c0e191 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23856" uid: 98096e03-8f4b-4c40-8539-47a7d15a509a kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Scaled up replica set error-404-isvc-8edb5-predictor-667c7d688c from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5-predictor.18a959f7a5018bcc namespace: kserve-ci-e2e-test resourceVersion: "23860" uid: 33218318-2ac3-4fba-b4c4-1d909aa7fa81 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23857" uid: 55337a25-b373-4bf8-ad15-c08086968ed8 kind: Event lastTimestamp: "2026-04-24T17:07:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:58Z" name: error-404-isvc-8edb5-predictor.18a959fb238cbd7f namespace: kserve-ci-e2e-test resourceVersion: "24211" uid: be0fc9f8-4ead-4b75-812b-c10059aaf031 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23857" uid: 55337a25-b373-4bf8-ad15-c08086968ed8 kind: Event lastTimestamp: "2026-04-24T17:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:58Z" name: error-404-isvc-8edb5-predictor.18a959fb23f64032 namespace: kserve-ci-e2e-test resourceVersion: "24212" uid: a539d76c-9f75-49c3-a64e-ccf4eb404823 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "24056" uid: 55337a25-b373-4bf8-ad15-c08086968ed8 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: error-404-isvc-8edb5-predictor.18a95a0221b700e5 namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 14f8f8ec-669b-45a8-950c-1993a99a969d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "24056" uid: 55337a25-b373-4bf8-ad15-c08086968ed8 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: error-404-isvc-8edb5-predictor.18a95a0222355962 namespace: kserve-ci-e2e-test resourceVersion: "24781" uid: bca3c370-37fa-43e7-83b1-0e886e9520ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: dc776a92-3c00-4001-9ea2-333cfcf6fb95 kind: Event lastTimestamp: "2026-04-24T17:08:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:16Z" name: error-404-isvc-8edb5.18a959f7a4d20789 namespace: kserve-ci-e2e-test resourceVersion: "24524" uid: 5bac2349-d3a6-4778-9e1c-27128f2116c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: dc776a92-3c00-4001-9ea2-333cfcf6fb95 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-8edb5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8edb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5.18a959f7a5862d32 namespace: kserve-ci-e2e-test resourceVersion: "23862" uid: 59544222-07a2-42c1-a5c1-61af180db989 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: dc776a92-3c00-4001-9ea2-333cfcf6fb95 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8edb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:28Z" name: error-404-isvc-8edb5.18a959f7a58674ba namespace: kserve-ci-e2e-test resourceVersion: "23863" uid: bc27c94d-7bc2-4fc9-9ff3-bba7de85da84 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24522" uid: dc776a92-3c00-4001-9ea2-333cfcf6fb95 kind: Event lastTimestamp: "2026-04-24T17:08:16Z" message: InferenceService [error-404-isvc-8edb5] is Ready metadata: creationTimestamp: "2026-04-24T17:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:16Z" name: error-404-isvc-8edb5.18a95a02d9f6a1a2 namespace: kserve-ci-e2e-test resourceVersion: "24523" uid: 5bb2608b-d9a1-476d-bf47-9557f4809846 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:08:02.610193Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24343" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9c25c-predictor-577bbdbf95-47mlg to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffabeadaf0 namespace: kserve-ci-e2e-test resourceVersion: "24347" uid: ed031e48-c1ae-44ea-ba8b-4a8d2ea9d11d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9c25c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffba0ba83e namespace: kserve-ci-e2e-test resourceVersion: "24357" uid: d9eb4cc7-1ca6-451e-95d3-0eb740e93ea9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24353" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffe90609dd namespace: kserve-ci-e2e-test resourceVersion: "24374" uid: 9835d7f2-cc7f-48b7-9e42-4f0b71702ed3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffea3f9dcd namespace: kserve-ci-e2e-test resourceVersion: "24379" uid: adf95950-6e40-4b59-97a0-a44a0c3012e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffed77a261 namespace: kserve-ci-e2e-test resourceVersion: "24384" uid: 3cf649e8-2501-41ff-a654-2f9e932c4424 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffede2f324 namespace: kserve-ci-e2e-test resourceVersion: "24386" uid: 478d3a16-958d-4c2b-a2b2-4c92706bec17 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959ffedee4297 namespace: kserve-ci-e2e-test resourceVersion: "24387" uid: f1c97364-fe64-4e27-98bb-9c2052e7ae89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959fff1832d77 namespace: kserve-ci-e2e-test resourceVersion: "24388" uid: 8f701428-84eb-4465-96ef-d5c1ff8d334e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a959fff1fbbaf9 namespace: kserve-ci-e2e-test resourceVersion: "24389" uid: 98004ff9-59d0-4cba-9d1d-90a06721eece reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:08:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:41Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a95a004cb8a066 namespace: kserve-ci-e2e-test resourceVersion: "24761" uid: 931d2f77-7c92-46bb-9297-3e2334f5fe86 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:17:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:17Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a95a80dc7d0db0 namespace: kserve-ci-e2e-test resourceVersion: "28741" uid: 3be2764c-d7c8-45bc-a4fe-5e6285af9fe4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: 427c28cc-938e-40b6-b4c0-28d6ac14c441 kind: Event lastTimestamp: "2026-04-24T17:17:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:17Z" name: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg.18a95a80dc7d8579 namespace: kserve-ci-e2e-test resourceVersion: "28742" uid: 6c978a4c-9e81-4dd6-85dd-f1c3ce0eecc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9c25c-predictor-577bbdbf95 namespace: kserve-ci-e2e-test resourceVersion: "24336" uid: 2d63c105-9a41-44ee-9697-59f0fecde9c7 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'Created pod: error-404-isvc-9c25c-predictor-577bbdbf95-47mlg' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c-predictor-577bbdbf95.18a959ffab7ed3a9 namespace: kserve-ci-e2e-test resourceVersion: "24344" uid: be178d3a-c9da-4ff5-bb86-e793046216de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: 8fdff288-447c-4058-8302-3b8f65fe3b90 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Scaled up replica set error-404-isvc-9c25c-predictor-577bbdbf95 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c-predictor.18a959ffa9cdf91a namespace: kserve-ci-e2e-test resourceVersion: "24338" uid: bf218916-0178-4fb1-8f42-4965587a7748 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24335" uid: d8193e5b-d6ee-4055-9a88-f3eefc2ac4d8 kind: Event lastTimestamp: "2026-04-24T17:08:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:47Z" name: error-404-isvc-9c25c-predictor.18a95a032845b556 namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: 24adea2a-e962-4a18-baab-5c6cfec76172 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24335" uid: d8193e5b-d6ee-4055-9a88-f3eefc2ac4d8 kind: Event lastTimestamp: "2026-04-24T17:08:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:47Z" name: error-404-isvc-9c25c-predictor.18a95a0328ad4986 namespace: kserve-ci-e2e-test resourceVersion: "24817" uid: b4db1277-c8ab-46a7-a863-895a1e05e60f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24540" uid: d8193e5b-d6ee-4055-9a88-f3eefc2ac4d8 kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: error-404-isvc-9c25c-predictor.18a95a0da4c3a3e6 namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 6509650b-4728-44d2-aac8-61e2ab5c4ebf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24540" uid: d8193e5b-d6ee-4055-9a88-f3eefc2ac4d8 kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: error-404-isvc-9c25c-predictor.18a95a0da52fc47f namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 0837dd44-a601-4b75-998b-353c3bacb710 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24303" uid: 480333a3-67ad-4999-87f5-61dfcfef4e26 kind: Event lastTimestamp: "2026-04-24T17:08:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:51Z" name: error-404-isvc-9c25c.18a959ffa9a81f84 namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: c6c1d083-09c2-4a58-aae0-9177fd773a0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24303" uid: 480333a3-67ad-4999-87f5-61dfcfef4e26 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-9c25c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c.18a959ffaa0c10a5 namespace: kserve-ci-e2e-test resourceVersion: "24340" uid: dbf5e837-e465-463d-8111-52047d693edc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24303" uid: 480333a3-67ad-4999-87f5-61dfcfef4e26 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:02Z" name: error-404-isvc-9c25c.18a959ffaa0c66a9 namespace: kserve-ci-e2e-test resourceVersion: "24341" uid: 5740b761-0dbf-4584-bbbf-4fcad9494a8b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: 480333a3-67ad-4999-87f5-61dfcfef4e26 kind: Event lastTimestamp: "2026-04-24T17:08:51Z" message: InferenceService [error-404-isvc-9c25c] is Ready metadata: creationTimestamp: "2026-04-24T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:51Z" name: error-404-isvc-9c25c.18a95a0b0b25d637 namespace: kserve-ci-e2e-test resourceVersion: "24858" uid: fdcf62b4-7f69-4cd0-9e65-bd79b250ff8b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:56:27.607326Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18179" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f6171-predictor-669d57f96f-22hp6 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:56:27Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595dda84b35b namespace: kserve-ci-e2e-test resourceVersion: "18188" uid: de5e0891-6926-42f7-98b0-6bc2083be552 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18196" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595df3c6be02 namespace: kserve-ci-e2e-test resourceVersion: "18221" uid: 6f4f2aa7-da59-4d43-afc8-9138e420774f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595df62c1a3b namespace: kserve-ci-e2e-test resourceVersion: "18224" uid: 4c99294b-e9fb-457a-a8c0-7f2116cb45f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595df964ddc5 namespace: kserve-ci-e2e-test resourceVersion: "18229" uid: 9a2c47e1-c3df-4ba1-bac8-0a106bd844ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595df9d4b580 namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: f3bb140f-84ed-4f65-93c6-6af0245c45ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595df9df8239 namespace: kserve-ci-e2e-test resourceVersion: "18231" uid: ac7f2ee1-126e-4367-86b2-0df44720e3b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595dfcb7b3ef namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: fab5220d-09bc-4e40-89a8-4326ee5eaed2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595dfd26c158 namespace: kserve-ci-e2e-test resourceVersion: "18235" uid: 434052df-1267-4fc1-b667-8057cdb888de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9595e31680fff namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: 4d6d02e6-c7be-4db5-92f4-b2dd8d27ad24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9596e25da25bb namespace: kserve-ci-e2e-test resourceVersion: "19083" uid: 0671e807-ddb7-4364-b8b4-759d22b08a94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9596e25da9a36 namespace: kserve-ci-e2e-test resourceVersion: "19087" uid: 4870b25f-fb73-48ca-be3a-8e0943f147d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6171-predictor-669d57f96f-22hp6 namespace: kserve-ci-e2e-test resourceVersion: "18181" uid: 78bbb2de-6a30-45fd-ac95-13a9331a5f04 kind: Event lastTimestamp: "2026-04-24T16:57:40Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:40Z" name: error-404-isvc-f6171-predictor-669d57f96f-22hp6.18a9596eb9a72cad namespace: kserve-ci-e2e-test resourceVersion: "19158" uid: cd5197e0-f12a-4fe4-890a-19ea289cfe50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f6171-predictor-669d57f96f namespace: kserve-ci-e2e-test resourceVersion: "18169" uid: b2663972-9c3c-466a-acc1-b1fcf0dd89c3 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: 'Created pod: error-404-isvc-f6171-predictor-669d57f96f-22hp6' metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:27Z" name: error-404-isvc-f6171-predictor-669d57f96f.18a9595dd94550ca namespace: kserve-ci-e2e-test resourceVersion: "18182" uid: 12ad6078-a1bf-4766-9a1c-1579cea44954 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18167" uid: 8724683c-2b9d-4995-9ec9-67d53bc5954c kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Scaled up replica set error-404-isvc-f6171-predictor-669d57f96f from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:27Z" name: error-404-isvc-f6171-predictor.18a9595dd5726e34 namespace: kserve-ci-e2e-test resourceVersion: "18173" uid: bf5dcc76-86d5-4edd-8598-572f6842b3f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18168" uid: 05b9993b-1a47-44bd-a0e3-4ea317218312 kind: Event lastTimestamp: "2026-04-24T16:56:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:57Z" name: error-404-isvc-f6171-predictor.18a9596154018625 namespace: kserve-ci-e2e-test resourceVersion: "18478" uid: 7077cb7e-58ca-4daf-aea3-f46bc6829c4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18168" uid: 05b9993b-1a47-44bd-a0e3-4ea317218312 kind: Event lastTimestamp: "2026-04-24T16:56:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:57Z" name: error-404-isvc-f6171-predictor.18a9596154adb857 namespace: kserve-ci-e2e-test resourceVersion: "18479" uid: fbf6aff7-c69e-4add-a94d-1b55fe24afec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: 05b9993b-1a47-44bd-a0e3-4ea317218312 kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: error-404-isvc-f6171-predictor.18a9596852ac8377 namespace: kserve-ci-e2e-test resourceVersion: "18911" uid: 58a4cae6-8cef-41eb-8017-8b57401995bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: 05b9993b-1a47-44bd-a0e3-4ea317218312 kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: error-404-isvc-f6171-predictor.18a95968531efe19 namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 9c52ccdb-3ee9-446e-95e5-4315a3568167 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 40661482-5c1d-4167-98da-7c61fbf294f3 kind: Event lastTimestamp: "2026-04-24T16:57:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:15Z" name: error-404-isvc-f6171.18a9595dd4f33a65 namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 985f8a84-e444-41cd-8ba7-425a9442baea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 40661482-5c1d-4167-98da-7c61fbf294f3 kind: Event lastTimestamp: "2026-04-24T16:57:15Z" message: InferenceService [error-404-isvc-f6171] is Ready metadata: creationTimestamp: "2026-04-24T16:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:15Z" name: error-404-isvc-f6171.18a95968ed5d5766 namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: e4f2ac94-32e5-4b29-a648-001197b61d6f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:45:37.913510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12618" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t to ip-10-0-143-104.ec2.internal metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:45:37Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c695bfd964 namespace: kserve-ci-e2e-test resourceVersion: "12623" uid: 8704b580-4264-4346-a176-30db49849176 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12633" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:45:38Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c6af80f7df namespace: kserve-ci-e2e-test resourceVersion: "12682" uid: 96ff8228-fcd3-41c5-926f-041b01d8c909 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:38Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c6b0e19d8f namespace: kserve-ci-e2e-test resourceVersion: "12686" uid: a07c8978-0473-4f33-a679-fdc83a6b5f09 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" in 4.276s (4.276s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c7afcafe17 namespace: kserve-ci-e2e-test resourceVersion: "12776" uid: b46d2a9c-2435-4d18-99f6-1ad47bce1170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c7b3c070ec namespace: kserve-ci-e2e-test resourceVersion: "12778" uid: 1d8bb2b2-a7b7-4330-980d-67498dc536e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c7b43660db namespace: kserve-ci-e2e-test resourceVersion: "12780" uid: d3204ca5-9248-4f5e-8c11-50b21610604c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:45:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1444" metadata: creationTimestamp: "2026-04-24T16:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:47Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958c8c27c7c03 namespace: kserve-ci-e2e-test resourceVersion: "12833" uid: 51b7a5ea-3631-4df0-8f10-bdf29f1b5ccb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1444" in 15.714s (15.714s including waiting). Image size: 1560917938 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:02Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cc6b23bb4b namespace: kserve-ci-e2e-test resourceVersion: "12997" uid: 6c97776f-f343-4ad0-bc6d-9a88cd3e6980 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cc6f236866 namespace: kserve-ci-e2e-test resourceVersion: "13001" uid: 449ca063-abb0-472b-a96f-ce3bf6d8fac8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cc6fb76425 namespace: kserve-ci-e2e-test resourceVersion: "13002" uid: a7d90af3-6698-43d2-a16b-fd1f9396a5d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cc6fc38fd6 namespace: kserve-ci-e2e-test resourceVersion: "13003" uid: 94c64c3d-ef36-4ad2-a09a-c7b6e4fa65e5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.44s (2.44s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cd01424b75 namespace: kserve-ci-e2e-test resourceVersion: "13023" uid: 9ceb75f9-d1a5-410f-b45b-1c0656040eb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cd056afe04 namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 7c7c6519-73a6-423b-bb95-366aaaebf4ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cd05e04483 namespace: kserve-ci-e2e-test resourceVersion: "13028" uid: 2504d1a8-c807-4337-8d7a-eed05035f1dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T16:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:47:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:03Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958cd717e3ced namespace: kserve-ci-e2e-test resourceVersion: "13632" uid: 6f1dbf91-295b-4eba-96cf-f8b08555c1d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:47Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958e4df583fa8 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 5d92f0cc-2e2a-446e-a7f0-f5452bfbe41e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958e4df58b6e9 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 53e60490-5e2b-4874-96e8-e92237af27e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t namespace: kserve-ci-e2e-test resourceVersion: "12620" uid: c05d0d59-f939-480f-8087-d416851e1cea kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t.18a958e4f582aeaa namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: e8582e13-7be8-4263-9d95-64c35f31a9cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-64bf5cc5fb namespace: kserve-ci-e2e-test resourceVersion: "12610" uid: 5364133d-930b-4b06-b020-21fa7ba1c7e2 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-64bf5cc5fb-67w2t' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: isvc-sklearn-graph-1-predictor-64bf5cc5fb.18a958c69554c5a7 namespace: kserve-ci-e2e-test resourceVersion: "12619" uid: 693fa4ae-d41a-4062-a075-3944b9a65995 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12608" uid: 514b9863-d3e4-4542-959b-929841448be6 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-64bf5cc5fb from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: isvc-sklearn-graph-1-predictor.18a958c6924f53d9 namespace: kserve-ci-e2e-test resourceVersion: "12612" uid: 190cea27-e714-4509-9413-7149674a6bae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12609" uid: 369470ba-7e77-42b2-b161-a66f6b39074f kind: Event lastTimestamp: "2026-04-24T16:46:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:07Z" name: isvc-sklearn-graph-1-predictor.18a958ca10f8f945 namespace: kserve-ci-e2e-test resourceVersion: "13055" uid: a106831e-1bd5-480d-8da3-2b222ffb285d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12609" uid: 369470ba-7e77-42b2-b161-a66f6b39074f kind: Event lastTimestamp: "2026-04-24T16:46:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:07Z" name: isvc-sklearn-graph-1-predictor.18a958ca1175547b namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: b4118be9-b99e-4b92-8278-79eef13dd7fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:46:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12889" uid: 369470ba-7e77-42b2-b161-a66f6b39074f kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:37Z" name: isvc-sklearn-graph-1-predictor.18a958d10f5d8c18 namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: 7bccace8-72ef-492f-bbd4-c981ae033c5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:46:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12889" uid: 369470ba-7e77-42b2-b161-a66f6b39074f kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:38Z" name: isvc-sklearn-graph-1-predictor.18a958d10fd99e1a namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: d89d0440-fa75-45cb-aaed-408357af67dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12584" uid: a686e0ff-5b45-4857-9c00-5723e80fb2c6 kind: Event lastTimestamp: "2026-04-24T16:47:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:14Z" name: isvc-sklearn-graph-1.18a958c6922b510e namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: 791dadb3-55f6-4b01-925b-ef2e84355ee9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: a686e0ff-5b45-4857-9c00-5723e80fb2c6 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:13Z" name: isvc-sklearn-graph-1.18a958dcda0a7ff1 namespace: kserve-ci-e2e-test resourceVersion: "13875" uid: 104cf1cc-d46b-4ef3-9cd2-e89dca025e6f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:45:38.921006Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12714" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-849696d87b-dzp6r to ip-10-0-143-104.ec2.internal metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:45:38Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c6d1cd0826 namespace: kserve-ci-e2e-test resourceVersion: "12718" uid: 09bb1502-6eaa-4891-ade5-7eeeb7fbccfb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12722" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:45:39Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c6ed493eec namespace: kserve-ci-e2e-test resourceVersion: "12735" uid: 3d83be8e-ae07-47b8-854f-74ab978e9fd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:39Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c6ef14910b namespace: kserve-ci-e2e-test resourceVersion: "12737" uid: 56f57dd8-635e-428c-97b8-e47074c4d56c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" in 3.237s (3.237s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c7b0127118 namespace: kserve-ci-e2e-test resourceVersion: "12777" uid: 251b2ad7-f47f-4b8f-8447-c8657bf2ca1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c7b3cfcb97 namespace: kserve-ci-e2e-test resourceVersion: "12779" uid: 40396a63-cc38-45a7-abe3-62560ab944cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c7b453a12f namespace: kserve-ci-e2e-test resourceVersion: "12781" uid: d7b51a8f-f889-45cd-94a0-4654b313a993 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:45:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1444" metadata: creationTimestamp: "2026-04-24T16:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:47Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958c8c28e9833 namespace: kserve-ci-e2e-test resourceVersion: "12834" uid: 455df717-b12b-4b4b-837d-6be5c371a920 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1444" in 15.689s (15.689s including waiting). Image size: 1560917938 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:02Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cc69b58a8b namespace: kserve-ci-e2e-test resourceVersion: "12996" uid: 804c16fd-7e7a-4fec-9bc2-d3f9492ba66d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cc6e87d95d namespace: kserve-ci-e2e-test resourceVersion: "12998" uid: 3b71f110-37a0-4653-b3e2-5d27bb459c95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cc6f014f4b namespace: kserve-ci-e2e-test resourceVersion: "12999" uid: 72a31fbc-8a17-4f18-ab6f-1d2f739b3fd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:03Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cc6f0f0358 namespace: kserve-ci-e2e-test resourceVersion: "13000" uid: c0c4cc50-6da4-4472-b2c6-40a0456e0fbe reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.458s (2.458s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cd019cf469 namespace: kserve-ci-e2e-test resourceVersion: "13025" uid: 88364322-5c44-4953-bfe8-23b46595bcd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cd0576c881 namespace: kserve-ci-e2e-test resourceVersion: "13027" uid: 4f352acd-9eec-4877-9ce9-90f269c6ab78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:46:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:05Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cd05eda60e namespace: kserve-ci-e2e-test resourceVersion: "13029" uid: 3444253d-c607-49e0-8bb2-2527addf949e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T16:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:47:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:03Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958cd717a684d namespace: kserve-ci-e2e-test resourceVersion: "13631" uid: b5180d52-c3e7-449e-b9b3-aae220905849 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:47Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958e4dd16bfe3 namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: 6ee8366a-89ff-487b-bec0-d576e6859ffd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:47Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958e4dd17c25c namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 8b7bb055-7d1a-4eb8-8687-8030b7dfcc95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r namespace: kserve-ci-e2e-test resourceVersion: "12716" uid: 2637fd52-945a-467d-8c18-3a8918b60faf kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r.18a958e4f5837ddd namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: b34cf8f1-898c-4343-93ff-1b57fc0e06bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-104.ec2.internal source: component: kubelet host: ip-10-0-143-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-849696d87b namespace: kserve-ci-e2e-test resourceVersion: "12699" uid: 1d5d4152-eb2e-43a4-9ff7-f0b8f6849e06 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-849696d87b-dzp6r' metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:38Z" name: isvc-sklearn-graph-2-predictor-849696d87b.18a958c6d14ef1f2 namespace: kserve-ci-e2e-test resourceVersion: "12715" uid: 40e6c2b1-2722-4068-b458-29cebb46c883 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12697" uid: 449cdc05-7886-4759-aa06-82eb574645d2 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-849696d87b from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:38Z" name: isvc-sklearn-graph-2-predictor.18a958c6baf4e9f8 namespace: kserve-ci-e2e-test resourceVersion: "12701" uid: eed6a7ce-0738-4731-ae71-a019d7a23f45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12698" uid: 67774f48-cab3-4039-b391-2b1671e71299 kind: Event lastTimestamp: "2026-04-24T16:46:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:08Z" name: isvc-sklearn-graph-2-predictor.18a958ca397d6a34 namespace: kserve-ci-e2e-test resourceVersion: "13071" uid: 3af6a757-f812-45c4-9962-fa608904bcdd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12698" uid: 67774f48-cab3-4039-b391-2b1671e71299 kind: Event lastTimestamp: "2026-04-24T16:46:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:08Z" name: isvc-sklearn-graph-2-predictor.18a958ca39ee2f32 namespace: kserve-ci-e2e-test resourceVersion: "13072" uid: 1871c52b-a278-4ef3-a52c-64e83b0fe4e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:46:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12902" uid: 67774f48-cab3-4039-b391-2b1671e71299 kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:38Z" name: isvc-sklearn-graph-2-predictor.18a958d13789b090 namespace: kserve-ci-e2e-test resourceVersion: "14122" uid: a21bc6ce-4706-4be0-aa91-f16536df2cfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:46:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12902" uid: 67774f48-cab3-4039-b391-2b1671e71299 kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:38Z" name: isvc-sklearn-graph-2-predictor.18a958d138141ed0 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: 7865bef5-4df2-47aa-b66d-2f3b77186522 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "12668" uid: eb65454f-a7af-41df-bad6-15e03be39dc7 kind: Event lastTimestamp: "2026-04-24T16:47:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:14Z" name: isvc-sklearn-graph-2.18a958c6bac91936 namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: 94b49e55-4079-47eb-8fe9-98e8ffd9770d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: eb65454f-a7af-41df-bad6-15e03be39dc7 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T16:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:14Z" name: isvc-sklearn-graph-2.18a958dcf1ec6a50 namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: b3296669-83ed-443e-ae5f-990ca0fd29f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:45:38.325122Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12675" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-jjzm9 to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:45:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c6ae488d49 namespace: kserve-ci-e2e-test resourceVersion: "12679" uid: ccd1cde8-f5bf-4074-af92-4ec1b71615f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c6b57ea8d3 namespace: kserve-ci-e2e-test resourceVersion: "12689" uid: 80118a9c-ad85-45e2-a403-22ef3e2e32d0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12683" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:45:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c6eb397046 namespace: kserve-ci-e2e-test resourceVersion: "12732" uid: 58c74d2c-7670-4107-a695-1192599ac5be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:39Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" metadata: creationTimestamp: "2026-04-24T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c6ecafc656 namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: eaf1fffc-7ec3-4d36-875e-e2247e4afbb0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:e180c5801eecc9f4bbb52efa5a04a7e91697942e5c91b352ed2867858a22963a" in 3.36s (3.36s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c7b5027380 namespace: kserve-ci-e2e-test resourceVersion: "12782" uid: 37de301a-1083-489f-9393-9f6bc59eb894 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c7b7e97ac7 namespace: kserve-ci-e2e-test resourceVersion: "12783" uid: 82f4d5cf-d924-4dbf-b131-1b740b492dff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c7b862c6f8 namespace: kserve-ci-e2e-test resourceVersion: "12784" uid: 1732d773-047e-496d-a655-acd1e0a2bc5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:45:47Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T16:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:47Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958c8d7dff54c namespace: kserve-ci-e2e-test resourceVersion: "12844" uid: aebd4c50-3b32-44e5-9447-50a175564e08 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:06Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.66s (18.66s including waiting). Image size: 1306417403 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cd30255f43 namespace: kserve-ci-e2e-test resourceVersion: "13034" uid: 73e65d80-918a-4303-8651-af325a7bd48c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cd333a2c1f namespace: kserve-ci-e2e-test resourceVersion: "13035" uid: 8ed10db8-845f-4439-b7e2-e15612a2e801 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cd33aefd1b namespace: kserve-ci-e2e-test resourceVersion: "13036" uid: ca96739a-f19f-44c6-b0d5-f0c81a97ea51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:06Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T16:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cd33bb117b namespace: kserve-ci-e2e-test resourceVersion: "13037" uid: 5f6f46dd-7e48-4747-9f98-f473e3505e23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:08Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.145s (2.145s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T16:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cdb39e7147 namespace: kserve-ci-e2e-test resourceVersion: "13066" uid: 261ffd74-e13b-49cd-9226-bcdf4f6c94bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cdb71864e5 namespace: kserve-ci-e2e-test resourceVersion: "13068" uid: 6e04a6ab-c17a-4bca-a712-5eb674f5da1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cdb7912c98 namespace: kserve-ci-e2e-test resourceVersion: "13069" uid: 9f822788-dcd9-4448-ace4-e9c36977d847 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:46:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:46:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958cdfb9c533a namespace: kserve-ci-e2e-test resourceVersion: "13538" uid: babc2f80-2836-439d-a099-e5a5da9c6a8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958e4e57b0037 namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: 73cc752f-27dc-4694-9b7b-16a44ee885d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958e4e57b6cfe namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 6d7f5f47-4e42-4678-ba4e-706698813d7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: ca4f3ee8-0572-4b8c-9308-badb39f9cea2 kind: Event lastTimestamp: "2026-04-24T16:47:50Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:50Z" name: isvc-xgboost-graph-predictor-669d8d6456-jjzm9.18a958e57fd5ddb8 namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: c64c5f94-36e9-4319-8d47-98794bc88e04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "12661" uid: deb325c6-e9d7-499d-8871-c5cda1db45e8 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-jjzm9' metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:38Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a958c6adb67bc3 namespace: kserve-ci-e2e-test resourceVersion: "12676" uid: f775974d-f816-46a6-861e-5cf9167534e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12658" uid: 155369bd-b9c7-48c2-ad98-703907b8f9f3 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:38Z" name: isvc-xgboost-graph-predictor.18a958c69e35f271 namespace: kserve-ci-e2e-test resourceVersion: "12664" uid: 83d7287c-da87-4b9f-932d-c69dc0c3574c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12660" uid: 35de1b3c-a8bb-48e2-bf16-d58c22570024 kind: Event lastTimestamp: "2026-04-24T16:46:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:38Z" name: isvc-xgboost-graph-predictor.18a958ca1cdcc21b namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 11dbd780-4860-4231-938f-d1497efa8d72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12660" uid: 35de1b3c-a8bb-48e2-bf16-d58c22570024 kind: Event lastTimestamp: "2026-04-24T16:46:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:38Z" name: isvc-xgboost-graph-predictor.18a958ca1d891285 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: 0aec92ee-f8f5-411b-81bc-260c31b019a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:46:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12897" uid: 35de1b3c-a8bb-48e2-bf16-d58c22570024 kind: Event lastTimestamp: "2026-04-24T16:47:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:23Z" name: isvc-xgboost-graph-predictor.18a958d11b6ce21d namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: ee891b95-796a-401a-aa76-1503ec61dfe9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:46:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12897" uid: 35de1b3c-a8bb-48e2-bf16-d58c22570024 kind: Event lastTimestamp: "2026-04-24T16:47:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:23Z" name: isvc-xgboost-graph-predictor.18a958d11bea066d namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 3d95c0ce-9e07-442d-bc9b-351427f83b89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12642" uid: 462de85f-d0de-4746-8253-f291449e0e21 kind: Event lastTimestamp: "2026-04-24T16:47:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:05Z" name: isvc-xgboost-graph.18a958c69e09ee5f namespace: kserve-ci-e2e-test resourceVersion: "13655" uid: a67a22f4-b8f0-4ada-a7f1-0991d4e0cbdf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: 462de85f-d0de-4746-8253-f291449e0e21 kind: Event lastTimestamp: "2026-04-24T16:47:05Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T16:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:05Z" name: isvc-xgboost-graph.18a958db0b7b7bdf namespace: kserve-ci-e2e-test resourceVersion: "13654" uid: 39bd8a7d-4ce3-48fe-ae7c-6ea172d4d6a5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:47:37.773934Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-787f8b7fc6-767sd to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:47:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:47:37Z" name: model-chainer-787f8b7fc6-767sd.18a958e27dfcbc57 namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: 97cae87b-2011-470c-9d9e-9ef104e62e76 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:37Z" name: model-chainer-787f8b7fc6-767sd.18a958e28a858a2e namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: 988afe3b-459d-4b21-b0b4-e22654bba7e1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14102" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:47:38Z" name: model-chainer-787f8b7fc6-767sd.18a958e2bac9c11c namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: 979494a5-f49c-4f4c-94ff-78b21eea7df0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:38Z" name: model-chainer-787f8b7fc6-767sd.18a958e2bc479692 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 4c9693e7-1881-498e-a430-9caff61dade4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:38Z" name: model-chainer-787f8b7fc6-767sd.18a958e2c004a103 namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: a2e78884-1bf7-4d71-8fc9-cbf6cc7e482b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:38Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:38Z" name: model-chainer-787f8b7fc6-767sd.18a958e2c08007b7 namespace: kserve-ci-e2e-test resourceVersion: "14130" uid: d87ca2a3-9d87-4ebf-9f99-06fbe5dfdbe7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:47Z" name: model-chainer-787f8b7fc6-767sd.18a958e4d686b494 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: be09f3e7-4c8c-45e8-bdf2-1e56b4a5b2ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-787f8b7fc6-767sd namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 8140754c-4236-412b-97ff-b990cd982c2f kind: Event lastTimestamp: "2026-04-24T16:48:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:14Z" name: model-chainer-787f8b7fc6-767sd.18a958e553bafb72 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 42bc4730-95b1-44c8-a51e-524a28139991 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-787f8b7fc6 namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: 0fcc42b5-7257-4e31-bcd3-9a8cd7105fa5 kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: 'Created pod: model-chainer-787f8b7fc6-767sd' metadata: creationTimestamp: "2026-04-24T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:37Z" name: model-chainer-787f8b7fc6.18a958e27d895c84 namespace: kserve-ci-e2e-test resourceVersion: "14097" uid: ccdedbd1-e5f4-44e6-94d3-25e1ee650e55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 91d0bd39-e93a-40f7-9341-9407d5454e8c kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: Scaled up replica set model-chainer-787f8b7fc6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:37Z" name: model-chainer.18a958e27c58f2d6 namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: 8e23ca21-c8a0-4712-a09d-b1e6d8758d13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14106" uid: 697fc770-0be9-405b-bf06-6100acec2f96 kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:37Z" name: model-chainer.18a958e2830db8c4 namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: ff687e0a-2250-43ba-92c1-82a120e41b01 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:48:48.054720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-30dd6-744fc64586-ptbhn to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:48:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:48:48Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f2db0c8217 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 29ac416f-1ce3-4afd-9a46-873616413d48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:48:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-30dd6-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:48Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f2eb539b42 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 94dd3cf1-d40a-4399-ac81-33e32979006e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:49Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:48:49Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:48:49Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f317e9570f namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 8171f89d-b8ad-47b0-8528-1ddcecde957b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30dd6} kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:48:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:49Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f3192d71b4 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 461ea0f2-df44-4a97-b837-e5ac94b822d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30dd6} kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:48:49Z" message: 'Created container: sequence-graph-30dd6' metadata: creationTimestamp: "2026-04-24T16:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:49Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f31c137fb5 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 776c2221-ebd3-42a8-b9ac-256823e5f0cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30dd6} kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:48:49Z" message: Started container sequence-graph-30dd6 metadata: creationTimestamp: "2026-04-24T16:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:49Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a958f31c89645d namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 220fb74e-2466-493a-9504-af324c6c5649 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30dd6} kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: Stopping container sequence-graph-30dd6 metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:02Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a959660ea42510 namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: d097a3ca-cc25-4f52-af08-696973621192 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30dd6} kind: Pod name: sequence-graph-30dd6-744fc64586-ptbhn namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cf9447c6-99a6-4598-a2cf-4f2419257de2 kind: Event lastTimestamp: "2026-04-24T16:57:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:30Z" name: sequence-graph-30dd6-744fc64586-ptbhn.18a9596699096c2b namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: 5eb4b2db-efcc-4cb0-9d39-f880e59f99e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-30dd6-744fc64586 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: b029ad96-6cb5-4608-b142-b5fc49b0947c kind: Event lastTimestamp: "2026-04-24T16:48:48Z" message: 'Created pod: sequence-graph-30dd6-744fc64586-ptbhn' metadata: creationTimestamp: "2026-04-24T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:48Z" name: sequence-graph-30dd6-744fc64586.18a958f2da7da38f namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 231c57d9-4423-4ea7-9928-4a79912a4257 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: c8b86da2-2869-46d1-9ebf-b801443f49bf kind: Event lastTimestamp: "2026-04-24T16:48:48Z" message: Scaled up replica set sequence-graph-30dd6-744fc64586 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:48Z" name: sequence-graph-30dd6.18a958f2d877f8e7 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: b28ac169-4de5-4745-9a26-41f144049d7c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 250ba485-8766-4f0a-ad24-1da92092d312 kind: Event lastTimestamp: "2026-04-24T16:48:48Z" message: InferenceGraph [sequence-graph-30dd6] is Ready metadata: creationTimestamp: "2026-04-24T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:48Z" name: sequence-graph-30dd6.18a958f2de7f5237 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 024097dd-cfdf-4f97-862d-b0a36244c1bd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: c62d5fd8-907c-4e58-afc5-b828c9981a5e kind: Event lastTimestamp: "2026-04-24T16:49:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:33Z" name: sequence-graph-30dd6.18a958f657f75f56 namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: cf5f239a-455e-4632-807d-b4e7f112d749 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: c62d5fd8-907c-4e58-afc5-b828c9981a5e kind: Event lastTimestamp: "2026-04-24T16:49:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:33Z" name: sequence-graph-30dd6.18a958f65899f2b6 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 13347226-42f9-4df2-ab0f-a6feb4443dff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:58:02.983395Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3eac5-77647df9db-6km8q to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:58:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:58:02Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a959740f6018d8 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 75c72342-e21a-4588-b5f7-1b52b52de578 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3eac5-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:03Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a959741753cb37 namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: f94b3e5a-9a95-4b88-b7d1-105540c6ba86 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:04Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:58:04Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a959744c271e88 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: f34952a7-af4e-450c-bd89-2273ca361f65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3eac5} kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:04Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a959744d7d41dd namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: f6969980-0a11-4794-bb75-f51016453ca9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3eac5} kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:04Z" message: 'Created container: sequence-graph-3eac5' metadata: creationTimestamp: "2026-04-24T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:04Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a95974503e2df8 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: a0661fb9-a3c8-4d6d-b26c-69c047b4f80d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3eac5} kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:04Z" message: Started container sequence-graph-3eac5 metadata: creationTimestamp: "2026-04-24T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:04Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a9597450b9036b namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: 40d3c99e-7cd1-4cf8-a01f-bb3f111c74c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3eac5} kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Stopping container sequence-graph-3eac5 metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a9597667a14f14 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a62c33bd-2592-40e6-9240-f06424e53317 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:58:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3eac5} kind: Pod name: sequence-graph-3eac5-77647df9db-6km8q namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d7a7516a-b5e9-4aad-9c46-29fd815b5171 kind: Event lastTimestamp: "2026-04-24T16:58:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:40Z" name: sequence-graph-3eac5-77647df9db-6km8q.18a95977081bcd0e namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 4285e1fe-aa56-479c-bfb4-ea75dc6bc1c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3eac5-77647df9db namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 0a1f6f1c-c860-48f7-a72e-29c176932dd1 kind: Event lastTimestamp: "2026-04-24T16:58:02Z" message: 'Created pod: sequence-graph-3eac5-77647df9db-6km8q' metadata: creationTimestamp: "2026-04-24T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:02Z" name: sequence-graph-3eac5-77647df9db.18a959740ee75ab2 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: 6aa408b5-5630-4599-945c-dbaac26f4e0f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: 90b83d13-63ab-40f7-a08a-daba00b864fe kind: Event lastTimestamp: "2026-04-24T16:58:02Z" message: Scaled up replica set sequence-graph-3eac5-77647df9db from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:02Z" name: sequence-graph-3eac5.18a959740e4648eb namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 11009010-a58f-40c2-b111-0e0677267d68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: 6ff64a5c-51ba-44f5-9c71-cf5d33a79083 kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: InferenceGraph [sequence-graph-3eac5] is Ready metadata: creationTimestamp: "2026-04-24T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:03Z" name: sequence-graph-3eac5.18a9597412cd5913 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: fffe69a8-e94e-43fb-8990-afa6b2328bc2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:59:13.311185Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-77cb4-7d9f7db785-bfm8l to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959846f3cf4ec namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: c2ca1c8c-545b-4c47-9b02-9088434138b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a95984884c5220 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 6a78f008-0696-42cf-83f9-aa8ef61ca0aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77cb4} kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959848abc3b77 namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 214f209c-8709-4d1a-b46e-b4de182cd1a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77cb4} kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: 'Created container: sequence-graph-77cb4' metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959848e37f7ff namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: e0ff16b0-f666-4c59-9211-197b41a8bba7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77cb4} kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: Started container sequence-graph-77cb4 metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959848eacc5ad namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 678f8e79-78fa-4f31-aa5b-0a67ad9a588c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77cb4} kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T17:07:27Z" message: Stopping container sequence-graph-77cb4 metadata: creationTimestamp: "2026-04-24T17:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:27Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959f79852b395 namespace: kserve-ci-e2e-test resourceVersion: "23787" uid: f71ed761-8a7e-4bb2-89eb-dd79c5837f8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77cb4} kind: Pod name: sequence-graph-77cb4-7d9f7db785-bfm8l namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: c4103ad4-10d3-4113-9913-dd967ce6405b kind: Event lastTimestamp: "2026-04-24T17:07:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T17:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:54Z" name: sequence-graph-77cb4-7d9f7db785-bfm8l.18a959f811a060ae namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: 805edfba-0c07-48b3-8782-766fd05d915b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-77cb4-7d9f7db785 namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: a3bf952a-981a-4daa-9223-2b328adb5e8d kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: 'Created pod: sequence-graph-77cb4-7d9f7db785-bfm8l' metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4-7d9f7db785.18a959846e6eb99d namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 26cccc6a-880b-4150-9ddf-c7015912b01a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 77a4aeff-ee8f-4b5e-8879-fddb3bff8509 kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: Scaled up replica set sequence-graph-77cb4-7d9f7db785 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4.18a959846d372242 namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: e7780cdc-c8ae-44b6-aa14-694171df2f08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: be6bf3a8-4b8d-48f3-b7b9-7073d430c552 kind: Event lastTimestamp: "2026-04-24T16:59:13Z" message: InferenceGraph [sequence-graph-77cb4] is Ready metadata: creationTimestamp: "2026-04-24T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:59:13Z" name: sequence-graph-77cb4.18a9598472b0b8d4 namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 2420ff9b-fee1-4cff-9324-3f9b8d197482 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 0f836949-58d0-427a-b5ac-1493aac37db5 kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: sequence-graph-77cb4.18a95987ec2496f4 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: a1182cd6-33f5-4d5d-9fa1-421b03f1579c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 0f836949-58d0-427a-b5ac-1493aac37db5 kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: sequence-graph-77cb4.18a95987ec6ecb26 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 04ad60c1-a186-4704-9d17-bb2e40ef2c1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 0f836949-58d0-427a-b5ac-1493aac37db5 kind: Event lastTimestamp: "2026-04-24T16:59:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:43Z" name: sequence-graph-77cb4.18a9598b6b3dcf99 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 873d7da8-7d77-405a-90c1-e07b9ec76240 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 0f836949-58d0-427a-b5ac-1493aac37db5 kind: Event lastTimestamp: "2026-04-24T16:59:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:43Z" name: sequence-graph-77cb4.18a9598b6bdea11d namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: ad779789-c833-4ebc-a1f0-349eea6423ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:07:52.296846Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-3827a-79444bd679-ncdhs to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959fd4531b645 namespace: kserve-ci-e2e-test resourceVersion: "24143" uid: fb1cf0fc-ae9e-4335-a311-2a5e56a1193a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24147" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959fd5e3fc35e namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: a0b5bab9-be0a-45e4-a04e-941086941c86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3827a} kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959fd5f6dfdee namespace: kserve-ci-e2e-test resourceVersion: "24163" uid: 11e53262-29a4-4c5f-92eb-a680e19efa77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3827a} kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'Created container: splitter-graph-3827a' metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959fd6289c9a4 namespace: kserve-ci-e2e-test resourceVersion: "24164" uid: 422c9e80-8ada-4166-b478-d16dab2bd417 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3827a} kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: Started container splitter-graph-3827a metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959fd6304ea9c namespace: kserve-ci-e2e-test resourceVersion: "24166" uid: 313c50df-b199-42e8-8921-a81c513e83fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3827a} kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Stopping container splitter-graph-3827a metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a959ff9d028880 namespace: kserve-ci-e2e-test resourceVersion: "24266" uid: 9d129bb5-55e6-42be-860d-2744f31ae39c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3827a} kind: Pod name: splitter-graph-3827a-79444bd679-ncdhs namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 9d298a1c-dd16-416a-82e4-347696632949 kind: Event lastTimestamp: "2026-04-24T17:08:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T17:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:29Z" name: splitter-graph-3827a-79444bd679-ncdhs.18a95a001982a438 namespace: kserve-ci-e2e-test resourceVersion: "24667" uid: f6a1d6e6-3cc6-41f4-98fa-a10a37192722 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-3827a-79444bd679 namespace: kserve-ci-e2e-test resourceVersion: "24135" uid: febb8e7f-ca4d-4f16-a3d0-6d394208f64e kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'Created pod: splitter-graph-3827a-79444bd679-ncdhs' metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a-79444bd679.18a959fd44cae228 namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: 2c40b2fd-6424-4102-9b2d-75a10fc16a92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-3827a namespace: kserve-ci-e2e-test resourceVersion: "24133" uid: 02852be5-ca22-4c6e-8b97-4c211721f211 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: Scaled up replica set splitter-graph-3827a-79444bd679 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a.18a959fd443697e4 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: f045d64e-45fc-4c68-b5f4-795815c5eb3b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3827a namespace: kserve-ci-e2e-test resourceVersion: "24150" uid: 78b4ca57-4b94-4e2c-8fdb-266df7ecda8c kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: InferenceGraph [splitter-graph-3827a] is Ready metadata: creationTimestamp: "2026-04-24T17:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:52Z" name: splitter-graph-3827a.18a959fd47de8ce2 namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 8b30585a-c763-42bf-96a6-136f493b0eb5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:09:02.570752Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9c25c-8bff57c84-thhml to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0da1d85a73 namespace: kserve-ci-e2e-test resourceVersion: "24961" uid: 27936578-d085-40a8-bc58-3dae7be5ebae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-9c25c-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0dada81e6e namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: 18ecf866-4010-4f16-aae3-39d5663aeda6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:03Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:09:03Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:09:03Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0ddec2c32d namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: ed66185c-76a4-434d-8f9b-892faee4cf51 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9c25c} kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:09:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T17:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:09:03Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0de001116b namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 700bb5d3-6377-4693-ac12-482a678db07c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9c25c} kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:09:03Z" message: 'Created container: splitter-graph-9c25c' metadata: creationTimestamp: "2026-04-24T17:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:09:03Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0de35260d7 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: 1e8f6964-5c91-48a8-b402-d1f56d0a8210 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9c25c} kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:09:03Z" message: Started container splitter-graph-9c25c metadata: creationTimestamp: "2026-04-24T17:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:09:03Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a0de3c90854 namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 74399676-4728-45cc-94ff-c3bd56c9428d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9c25c} kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:17:17Z" message: Stopping container splitter-graph-9c25c metadata: creationTimestamp: "2026-04-24T17:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:17Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a80d1cea55d namespace: kserve-ci-e2e-test resourceVersion: "28711" uid: 7991e2a9-b433-479a-bb92-31e364769ca6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9c25c} kind: Pod name: splitter-graph-9c25c-8bff57c84-thhml namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 4138b7fb-1a27-4eeb-8100-8e5ff46d968e kind: Event lastTimestamp: "2026-04-24T17:17:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T17:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:45Z" name: splitter-graph-9c25c-8bff57c84-thhml.18a95a819a272a0f namespace: kserve-ci-e2e-test resourceVersion: "28956" uid: eec8ff58-58a8-4f69-bb3f-a5ce33963e88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9c25c-8bff57c84 namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: de221228-8bed-4948-8b3a-6bcdf57dc6f4 kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: 'Created pod: splitter-graph-9c25c-8bff57c84-thhml' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c-8bff57c84.18a95a0da12c1e51 namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: 416196bf-b667-4e16-8000-3f3c808d3b26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24950" uid: c39adc1d-8362-4ed4-a927-4035d66e4f42 kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: Scaled up replica set splitter-graph-9c25c-8bff57c84 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c.18a95a0da0209581 namespace: kserve-ci-e2e-test resourceVersion: "24953" uid: 6fd09143-558e-4185-ad8e-97fc3a2fb0a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: 393ae4e9-187b-4a25-9388-7de1e2f2f055 kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: InferenceGraph [splitter-graph-9c25c] is Ready metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c.18a95a0da6346c44 namespace: kserve-ci-e2e-test resourceVersion: "24972" uid: 4eba620a-359a-4e22-8168-264e8d5bc90c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24941" uid: 393ae4e9-187b-4a25-9388-7de1e2f2f055 kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: 'Failed to update status for InferenceGraph "splitter-graph-9c25c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c.18a95a0da8a640d6 namespace: kserve-ci-e2e-test resourceVersion: "24973" uid: a96c74ba-7641-4991-8675-5bb92ba10121 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24941" uid: 393ae4e9-187b-4a25-9388-7de1e2f2f055 kind: Event lastTimestamp: "2026-04-24T17:09:02Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:09:02Z" name: splitter-graph-9c25c.18a95a0da8a68946 namespace: kserve-ci-e2e-test resourceVersion: "24974" uid: c9e302f3-cb75-48a9-9542-a760bd789730 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 28db7067-1067-4b7f-85e5-2cec42e28379 kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: splitter-graph-9c25c.18a95a111e8d77fc namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: 98d8c20b-5c0b-4a0e-b535-a0a4e4342216 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:09:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 28db7067-1067-4b7f-85e5-2cec42e28379 kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: splitter-graph-9c25c.18a95a111eeddd2c namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: e567de06-92c1-4755-be28-dc86868ea0ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:09:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: 28db7067-1067-4b7f-85e5-2cec42e28379 kind: Event lastTimestamp: "2026-04-24T17:09:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:47Z" name: splitter-graph-9c25c.18a95a149d981537 namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: 28071ae5-6d86-41fd-b03d-8309469ab6d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:09:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9c25c namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: 28db7067-1067-4b7f-85e5-2cec42e28379 kind: Event lastTimestamp: "2026-04-24T17:09:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:47Z" name: splitter-graph-9c25c.18a95a149dedf046 namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: f8bbfed5-3fee-4db5-a26f-def5e161b6fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:47:12.162984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dc8774ad96 namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: 51eb7185-1e0e-487c-afc6-8a2000f3c4e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13797" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca066d78f namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: bc21dd33-dbba-4601-9e95-2a8d0ebb9f96 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca1cbc26a namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 4d5e3bce-ed69-4dd7-b2c1-3568e3824386 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca479cf27 namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: d85bbc8a-6cf0-498b-9ef6-8b8e3a3e6168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca4dc7763 namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: d834b99e-f654-4661-86f0-2e93f7ca6049 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca4e69ba1 namespace: kserve-ci-e2e-test resourceVersion: "13833" uid: ebcff8d4-4e14-48bd-b3c2-a69b85dca1ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca760dd10 namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: 57ce8c68-15f4-4c19-ad35-24a5e1f35a91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dca7c402f6 namespace: kserve-ci-e2e-test resourceVersion: "13835" uid: e311544c-50b2-415a-afd6-62f64893f41e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:47:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:50Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a958dd095d139e namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 818af4ae-e0c5-45bb-a380-4a87b311322c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:27Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a9595dc5039a63 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: d689f189-ebef-4a7b-b6a8-8065df3b18c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 0352dfc6-f89c-4592-899c-58affbababb4 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:27Z" name: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4.18a9595dc5042a37 namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 4b60faa0-c1b7-4bba-bc1a-bfdb594a9a67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2c4dd-predictor-587bf956c8 namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 4f72db77-879f-4300-9f86-451766bff6ab kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Created pod: success-200-isvc-2c4dd-predictor-587bf956c8-px6z4' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor-587bf956c8.18a958dc86f5e0f2 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: bf998d51-952f-4c93-a4af-bfd4d058b2dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: 917e1528-e1a6-4578-8835-ce9f8c1360fd kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Scaled up replica set success-200-isvc-2c4dd-predictor-587bf956c8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd-predictor.18a958dc81a2baf7 namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 2a85b66b-c5e2-4283-ab3a-7be4739e7248 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: e54ce9b3-0bfa-42d0-8a4b-4c85ccc405f0 kind: Event lastTimestamp: "2026-04-24T16:47:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:57Z" name: success-200-isvc-2c4dd-predictor.18a958e0012412f6 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: d90631cc-0194-4e11-a10c-44232a09cdc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: e54ce9b3-0bfa-42d0-8a4b-4c85ccc405f0 kind: Event lastTimestamp: "2026-04-24T16:47:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:57Z" name: success-200-isvc-2c4dd-predictor.18a958e001ae9feb namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 81df7b30-37fe-4f76-9e8c-c1889ba3be50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: e54ce9b3-0bfa-42d0-8a4b-4c85ccc405f0 kind: Event lastTimestamp: "2026-04-24T16:48:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:27Z" name: success-200-isvc-2c4dd-predictor.18a958ea7f49f624 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: ae9f8b77-fb30-480f-a5b0-1de5a0fa740a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c4dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: e54ce9b3-0bfa-42d0-8a4b-4c85ccc405f0 kind: Event lastTimestamp: "2026-04-24T16:48:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:27Z" name: success-200-isvc-2c4dd-predictor.18a958ea7fdd4fe6 namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: a21a130e-314b-41b4-b5aa-150d3cc08644 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 2616b05c-d0a2-4f72-bf72-666c616bf264 kind: Event lastTimestamp: "2026-04-24T16:48:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:00Z" name: success-200-isvc-2c4dd.18a958dc814fa0ee namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 0f08cdd4-fd36-4c97-af53-2b1f73639329 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 2616b05c-d0a2-4f72-bf72-666c616bf264 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-2c4dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2c4dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd.18a958dc826210b5 namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 79598a08-0269-4efa-9b64-3a2783677302 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 2616b05c-d0a2-4f72-bf72-666c616bf264 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2c4dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-2c4dd.18a958dc82627d25 namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: be041466-9403-402c-b995-3e1c71dd68bb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 2616b05c-d0a2-4f72-bf72-666c616bf264 kind: Event lastTimestamp: "2026-04-24T16:48:00Z" message: InferenceService [success-200-isvc-2c4dd] is Ready metadata: creationTimestamp: "2026-04-24T16:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:00Z" name: success-200-isvc-2c4dd.18a958e7c43bce5e namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 7e15abfd-e362-4f15-8552-e00d3ca50fa2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:47:48.009898Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-30dd6-predictor-94645b55d-96c4f to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:47:48Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e4e01935bb namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 8468765a-f8b2-47b7-ba12-6e03051a3ed2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-30dd6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:48Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e4e81d851f namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 68a21f71-5f87-4a0f-a075-b19417a140d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e51cbf7771 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: aecd586e-95bc-47ea-baaa-766ec4bcdfcd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e51e16f886 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: d9366219-1128-449c-8f01-7b109a1a20ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e521212c47 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: dc765525-1842-48ca-8fe4-99e10315fca9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e5218daa26 namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 319fb23d-daaa-4218-b590-2f4ae810bb9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e521992522 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 13d79a7e-96e4-4839-b608-ae05f7fbdfa5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e524d84f99 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 7347c8c5-f5ce-4f6e-ba8c-63159d849192 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:47:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:49Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e52558bbbd namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 00c75f22-1fb7-4efd-abb7-f0e3fc01c2df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:48:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:26Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a958e5724a839b namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 10a3a8c3-38f4-45f7-bd9b-ecdd7f85dc48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a9596616523b6e namespace: kserve-ci-e2e-test resourceVersion: "18565" uid: f98a0276-6f20-4f34-966b-2eeac4ad4913 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30dd6-predictor-94645b55d-96c4f namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: b5b245aa-cb3e-44ab-8c6e-8d8cf0d3a0c0 kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-30dd6-predictor-94645b55d-96c4f.18a959661652c701 namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: 8bfe9ab3-164d-480e-8554-5f28aeb3f691 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-30dd6-predictor-94645b55d namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 4904308c-9776-4413-869c-4a540fdea347 kind: Event lastTimestamp: "2026-04-24T16:47:48Z" message: 'Created pod: success-200-isvc-30dd6-predictor-94645b55d-96c4f' metadata: creationTimestamp: "2026-04-24T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:48Z" name: success-200-isvc-30dd6-predictor-94645b55d.18a958e4df966cbe namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 91deb3a0-a277-4e27-8e17-c7152a8da1b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: bf5b9589-f75c-40a7-b37e-be20fbdf82b4 kind: Event lastTimestamp: "2026-04-24T16:47:47Z" message: Scaled up replica set success-200-isvc-30dd6-predictor-94645b55d from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:47Z" name: success-200-isvc-30dd6-predictor.18a958e4dda97024 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 8be6808b-a73d-4ce7-ae81-d74ad388548d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: f06bf3b0-ed12-4cd0-bb3b-e7fa099f7a7b kind: Event lastTimestamp: "2026-04-24T16:48:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:02Z" name: success-200-isvc-30dd6-predictor.18a958e85c3077c7 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: b73e0864-2f01-494f-9668-e58927133ca6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: f06bf3b0-ed12-4cd0-bb3b-e7fa099f7a7b kind: Event lastTimestamp: "2026-04-24T16:48:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:02Z" name: success-200-isvc-30dd6-predictor.18a958e85c7a618f namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 36234127-dfd0-4022-aed5-49916d4ab31a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:48:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: f06bf3b0-ed12-4cd0-bb3b-e7fa099f7a7b kind: Event lastTimestamp: "2026-04-24T16:49:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:03Z" name: success-200-isvc-30dd6-predictor.18a958ebdb427032 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: b8649ca1-b9ad-4d28-a778-ebfd0d8335f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T16:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: f06bf3b0-ed12-4cd0-bb3b-e7fa099f7a7b kind: Event lastTimestamp: "2026-04-24T16:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:49:03Z" name: success-200-isvc-30dd6-predictor.18a958ebdbcda232 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 1c0b88e9-f1af-4ecc-a0a9-297a1959bb20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: b31d6279-0034-4179-9d0b-22a22c3be064 kind: Event lastTimestamp: "2026-04-24T16:48:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:36Z" name: success-200-isvc-30dd6.18a958e4dd90be69 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 6ff62572-1c1c-449e-a801-b21cfa4ed61d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30dd6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: b31d6279-0034-4179-9d0b-22a22c3be064 kind: Event lastTimestamp: "2026-04-24T16:48:36Z" message: InferenceService [success-200-isvc-30dd6] is Ready metadata: creationTimestamp: "2026-04-24T16:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:36Z" name: success-200-isvc-30dd6.18a958f02fb7b31f namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 32d54d86-51b7-4bcb-90a2-7949b6d0d01d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:45:37.690100Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12583" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32209-predictor-649bfcd444-vqfb8 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958c6886edf14 namespace: kserve-ci-e2e-test resourceVersion: "12589" uid: b69ce922-083b-480c-85a6-bf66e2260619 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-32209-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958c693f26247 namespace: kserve-ci-e2e-test resourceVersion: "12616" uid: 86274fb2-c00e-4ad3-9cca-62461b0f7a3c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12595" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:45:38Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958c6c5e7f703 namespace: kserve-ci-e2e-test resourceVersion: "12708" uid: f61070be-288f-47ff-863f-eade17689002 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:38Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" metadata: creationTimestamp: "2026-04-24T16:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:38Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958c6c728986e namespace: kserve-ci-e2e-test resourceVersion: "12710" uid: ceb8d54c-4416-4f4b-b0bb-7fbb243e8fa5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" in 15.602s (15.602s including waiting). Image size: 1335706220 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958ca6923f2a3 namespace: kserve-ci-e2e-test resourceVersion: "12910" uid: 3bb42c27-a6bb-43af-98f5-6a16dbbb60e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958ca6cf7af27 namespace: kserve-ci-e2e-test resourceVersion: "12913" uid: ec9e87c7-1ecf-4d6c-83ee-93ea1502dbb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958ca6d67d2ab namespace: kserve-ci-e2e-test resourceVersion: "12915" uid: bd32b973-2b01-4f97-8330-3ae2e69d6182 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:54Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958ca6d738266 namespace: kserve-ci-e2e-test resourceVersion: "12916" uid: 660c2ddc-df56-451f-8c1e-e7fda19b55b9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.525s (2.525s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T16:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:56Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958cb03f6f2f3 namespace: kserve-ci-e2e-test resourceVersion: "12937" uid: ff1d23fa-1d3f-40de-8d20-e41b3e345e68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:57Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958cb088d7cdb namespace: kserve-ci-e2e-test resourceVersion: "12939" uid: 40c5739e-1d36-465a-9041-feac42397339 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:45:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:45:57Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958cb0904af6a namespace: kserve-ci-e2e-test resourceVersion: "12941" uid: 594b677e-fd82-4292-8f61-4175d444059a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T16:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958cb0d143684 namespace: kserve-ci-e2e-test resourceVersion: "13844" uid: 23f6b469-078d-426d-816f-82bdf2561e52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958dc82175494 namespace: kserve-ci-e2e-test resourceVersion: "13760" uid: c8f0d75f-22cd-45b0-8ea7-dca51b1b15db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:47:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:12Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958dc8217ca80 namespace: kserve-ci-e2e-test resourceVersion: "13763" uid: bf9437de-a49d-4585-a3e0-c5e63e4c4c8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32209-predictor-649bfcd444-vqfb8 namespace: kserve-ci-e2e-test resourceVersion: "12587" uid: 9b01fef6-6066-43b8-a153-e14d7d125268 kind: Event lastTimestamp: "2026-04-24T16:47:13Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:13Z" name: success-200-isvc-32209-predictor-649bfcd444-vqfb8.18a958dcbf5267f0 namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: d588ea18-ffcf-4b02-b619-fe242e0ecb89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32209-predictor-649bfcd444 namespace: kserve-ci-e2e-test resourceVersion: "12571" uid: 9637a03e-8336-453c-b3f4-7f38bb2d8226 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'Created pod: success-200-isvc-32209-predictor-649bfcd444-vqfb8' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209-predictor-649bfcd444.18a958c68810399e namespace: kserve-ci-e2e-test resourceVersion: "12586" uid: 495e576f-4184-486e-9983-82076fc38124 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12570" uid: 92e940fd-aa13-48c4-9e55-9428c0cbd7f7 kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: Scaled up replica set success-200-isvc-32209-predictor-649bfcd444 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209-predictor.18a958c68088f7c0 namespace: kserve-ci-e2e-test resourceVersion: "12572" uid: 8ac83875-d48f-461f-955d-cc50062a9970 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12579" uid: 650726c9-f086-4dc4-817b-f90ea3d4de43 kind: Event lastTimestamp: "2026-04-24T16:45:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:52Z" name: success-200-isvc-32209-predictor.18a958ca05e18d97 namespace: kserve-ci-e2e-test resourceVersion: "12884" uid: 91ad05d6-7062-483f-b1a8-2ca7102acfc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12579" uid: 650726c9-f086-4dc4-817b-f90ea3d4de43 kind: Event lastTimestamp: "2026-04-24T16:45:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:45:52Z" name: success-200-isvc-32209-predictor.18a958ca06509023 namespace: kserve-ci-e2e-test resourceVersion: "12885" uid: 3d7f4ab6-37dd-46bd-a812-c38a8f03b05e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12883" uid: 650726c9-f086-4dc4-817b-f90ea3d4de43 kind: Event lastTimestamp: "2026-04-24T16:47:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:07Z" name: success-200-isvc-32209-predictor.18a958cd86035c0d namespace: kserve-ci-e2e-test resourceVersion: "13668" uid: 874a2cff-bfdb-4ff3-8293-6a83ce48cd92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32209-predictor namespace: kserve-ci-e2e-test resourceVersion: "12883" uid: 650726c9-f086-4dc4-817b-f90ea3d4de43 kind: Event lastTimestamp: "2026-04-24T16:47:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:47:07Z" name: success-200-isvc-32209-predictor.18a958cd8695e5cb namespace: kserve-ci-e2e-test resourceVersion: "13669" uid: 44140c1d-2adc-42a5-96d7-2a0bccfd1e9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "12560" uid: 571d6f02-fb75-40b9-b814-f5b5c743201c kind: Event lastTimestamp: "2026-04-24T16:46:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:46:43Z" name: success-200-isvc-32209.18a958c6867586bb namespace: kserve-ci-e2e-test resourceVersion: "13433" uid: c6a527b8-cf57-4513-96f6-7cfc3821a401 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "12560" uid: 571d6f02-fb75-40b9-b814-f5b5c743201c kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-32209": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209.18a958c686c46b99 namespace: kserve-ci-e2e-test resourceVersion: "12581" uid: 6b6e4bf5-a825-4a95-b00a-e573403f6eb6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "12560" uid: 571d6f02-fb75-40b9-b814-f5b5c743201c kind: Event lastTimestamp: "2026-04-24T16:45:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:45:37Z" name: success-200-isvc-32209.18a958c686c4b7eb namespace: kserve-ci-e2e-test resourceVersion: "12582" uid: da364793-eb04-4e53-b48a-2526612c7079 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32209 namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 571d6f02-fb75-40b9-b814-f5b5c743201c kind: Event lastTimestamp: "2026-04-24T16:46:43Z" message: InferenceService [success-200-isvc-32209] is Ready metadata: creationTimestamp: "2026-04-24T16:46:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:46:43Z" name: success-200-isvc-32209.18a958d5cbb43987 namespace: kserve-ci-e2e-test resourceVersion: "13428" uid: 52da832e-27ce-4608-93cf-bd66865222ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:06:52.253516Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23419" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3827a-predictor-7f98b8946b-25n6x to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef4a553b69 namespace: kserve-ci-e2e-test resourceVersion: "23429" uid: a4b95ddf-ae4b-4b34-ae01-b4174a0c79c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23441" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef6348f0b3 namespace: kserve-ci-e2e-test resourceVersion: "23466" uid: 408bbbee-4af3-4999-931a-ba220193eb9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef649c409b namespace: kserve-ci-e2e-test resourceVersion: "23468" uid: c8f6cbfd-b88a-4790-ad14-ab970726ce39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef6773a872 namespace: kserve-ci-e2e-test resourceVersion: "23469" uid: 1438f151-c9c5-4ac3-8a74-c62a87929b86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef67f388a8 namespace: kserve-ci-e2e-test resourceVersion: "23471" uid: 6643991c-fe91-4285-9545-43710d6bd275 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef67fff526 namespace: kserve-ci-e2e-test resourceVersion: "23472" uid: 5cde4c8f-3a6e-4a56-bf6e-0d07d90be4b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef6b89525f namespace: kserve-ci-e2e-test resourceVersion: "23476" uid: 0ca426dc-1a4d-49f3-b62b-4e144cfd7387 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef6bf7e312 namespace: kserve-ci-e2e-test resourceVersion: "23477" uid: f5c8813a-a437-44a5-b751-893bfd82bdac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:07:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:29Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ef797aebdd namespace: kserve-ci-e2e-test resourceVersion: "23906" uid: 62dc1325-8a58-48dd-af46-699a069a3b78 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ffa3a1e880 namespace: kserve-ci-e2e-test resourceVersion: "24300" uid: 69257163-9505-4ea0-8d43-310a3653bd90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a959ffa3a4d967 namespace: kserve-ci-e2e-test resourceVersion: "24301" uid: efef780b-80aa-4c0d-8a38-577a30726600 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x namespace: kserve-ci-e2e-test resourceVersion: "23423" uid: ae47a928-b7d6-4a05-9a47-2cf95df423ca kind: Event lastTimestamp: "2026-04-24T17:08:04Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:04Z" name: success-200-isvc-3827a-predictor-7f98b8946b-25n6x.18a95a0001ba87c2 namespace: kserve-ci-e2e-test resourceVersion: "24392" uid: 80e7c103-f842-4760-b8e6-5dfb516cb076 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3827a-predictor-7f98b8946b namespace: kserve-ci-e2e-test resourceVersion: "23398" uid: db34f58a-b6f7-4446-acbd-bccfc4aee6d0 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Created pod: success-200-isvc-3827a-predictor-7f98b8946b-25n6x' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor-7f98b8946b.18a959ef49e35bec namespace: kserve-ci-e2e-test resourceVersion: "23422" uid: 4465b2d5-67cc-457d-b882-2d98f397bdfb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23395" uid: 40cd5be7-e658-4ee5-bba1-08ba07202dc4 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Scaled up replica set success-200-isvc-3827a-predictor-7f98b8946b from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a-predictor.18a959ef4685a8d0 namespace: kserve-ci-e2e-test resourceVersion: "23401" uid: fdaf3995-3578-47b2-9422-1e6a611a756c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23397" uid: a8af91e0-9796-42c1-b3e2-2b15387fe559 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: success-200-isvc-3827a-predictor.18a959f2c505c4d2 namespace: kserve-ci-e2e-test resourceVersion: "24127" uid: 171844fc-a055-4874-9673-3c2f8bb2445e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23397" uid: a8af91e0-9796-42c1-b3e2-2b15387fe559 kind: Event lastTimestamp: "2026-04-24T17:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:52Z" name: success-200-isvc-3827a-predictor.18a959f2c55bf15c namespace: kserve-ci-e2e-test resourceVersion: "24130" uid: de93da13-2fb9-4638-bb00-fce58e920a31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23609" uid: a8af91e0-9796-42c1-b3e2-2b15387fe559 kind: Event lastTimestamp: "2026-04-24T17:07:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:22Z" name: success-200-isvc-3827a-predictor.18a959f644087d8d namespace: kserve-ci-e2e-test resourceVersion: "23721" uid: 0b05bc1e-2499-4ecb-bcb4-df0d01f4b7b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3827a-predictor namespace: kserve-ci-e2e-test resourceVersion: "23609" uid: a8af91e0-9796-42c1-b3e2-2b15387fe559 kind: Event lastTimestamp: "2026-04-24T17:07:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:22Z" name: success-200-isvc-3827a-predictor.18a959f64472190d namespace: kserve-ci-e2e-test resourceVersion: "23723" uid: 961b60fb-df67-4b25-830a-9aa214f58dab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: f26a19e7-5937-4d6e-82d9-0559b5657dd4 kind: Event lastTimestamp: "2026-04-24T17:07:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:39Z" name: success-200-isvc-3827a.18a959ef465abbc4 namespace: kserve-ci-e2e-test resourceVersion: "24017" uid: ed7a0e95-ada4-459e-bd95-7fa07a2e5038 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: f26a19e7-5937-4d6e-82d9-0559b5657dd4 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-3827a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3827a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a.18a959ef46ab33ee namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: 270f7c93-0d44-42cf-bb30-0af494c8b175 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: f26a19e7-5937-4d6e-82d9-0559b5657dd4 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3827a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-3827a.18a959ef46ab8751 namespace: kserve-ci-e2e-test resourceVersion: "23404" uid: f45f0ee7-7394-4d61-b03d-9213bf636f56 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3827a namespace: kserve-ci-e2e-test resourceVersion: "24014" uid: f26a19e7-5937-4d6e-82d9-0559b5657dd4 kind: Event lastTimestamp: "2026-04-24T17:07:39Z" message: InferenceService [success-200-isvc-3827a] is Ready metadata: creationTimestamp: "2026-04-24T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:39Z" name: success-200-isvc-3827a.18a959fa34ab7e4f namespace: kserve-ci-e2e-test resourceVersion: "24015" uid: 91b18ddd-5fd0-4861-ac81-62a122978053 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:57:03.006099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18568" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:57:03Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a95966187321d4 namespace: kserve-ci-e2e-test resourceVersion: "18573" uid: 58c07849-ddfa-43f9-a49b-ef08dfd00655 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3eac5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:03Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a9596627a6dff6 namespace: kserve-ci-e2e-test resourceVersion: "18630" uid: b8b40bed-65b7-47ba-a98e-30fe35d1d67d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18582" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:57:04Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959668acf16e1 namespace: kserve-ci-e2e-test resourceVersion: "18654" uid: c09cff75-2d71-4aad-908b-4d744db2cb00 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:04Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959668c77aff1 namespace: kserve-ci-e2e-test resourceVersion: "18656" uid: ec4974f9-6a0b-44bf-b313-3b210234585d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959668fbc18e0 namespace: kserve-ci-e2e-test resourceVersion: "18658" uid: e1960348-8d8c-46db-9a53-d18ca5f39717 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a9596690263d53 namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: ec79edae-0ccb-49af-bcdd-ceaf9dc50092 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959669030b472 namespace: kserve-ci-e2e-test resourceVersion: "18660" uid: d60ae88f-505c-4c25-bd23-86511c98f36c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a9596692c58e8b namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: e1b02e09-88e1-4ad7-b377-f8abf5ab2259 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959669331333f namespace: kserve-ci-e2e-test resourceVersion: "18662" uid: c8c6c581-f181-4110-836e-b6865ff33302 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:57:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:41Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a959669b03debf namespace: kserve-ci-e2e-test resourceVersion: "19170" uid: f57a7c42-afc9-479c-924b-fb52da7237e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a95976721a2ad1 namespace: kserve-ci-e2e-test resourceVersion: "19555" uid: fbd6e6d7-f615-4319-83af-e2e006f04e84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a95976721aa462 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: e6a911f2-2c7b-4be1-b3fa-e93f059f4024 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 9e8d6087-9ff2-4eb9-b90f-ae8f84bd9cc2 kind: Event lastTimestamp: "2026-04-24T16:58:16Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:58:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:16Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6.18a95977232e5a58 namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: 1ec815d9-61d0-41aa-80af-7a3efc13fce1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3eac5-predictor-5dcfddbf4b namespace: kserve-ci-e2e-test resourceVersion: "18555" uid: 98ed5ea9-bcf0-4d6d-ba82-1d8798210f1f kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: 'Created pod: success-200-isvc-3eac5-predictor-5dcfddbf4b-456v6' metadata: creationTimestamp: "2026-04-24T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-3eac5-predictor-5dcfddbf4b.18a9596617f42960 namespace: kserve-ci-e2e-test resourceVersion: "18570" uid: c9acd666-756e-495e-9e1b-e0f64702b2c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18547" uid: 334f85c8-e3ef-4581-9083-44c8145a9bea kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: Scaled up replica set success-200-isvc-3eac5-predictor-5dcfddbf4b from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-3eac5-predictor.18a95966152a1afb namespace: kserve-ci-e2e-test resourceVersion: "18560" uid: 6fb3b803-0bed-402e-8fb6-ffdfad364acc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18554" uid: d693ccd9-e78c-4c17-abba-e9af253e517c kind: Event lastTimestamp: "2026-04-24T16:57:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:48Z" name: success-200-isvc-3eac5-predictor.18a959699449e254 namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: 8efd914e-fb26-4e25-a296-6204cb0ac776 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18554" uid: d693ccd9-e78c-4c17-abba-e9af253e517c kind: Event lastTimestamp: "2026-04-24T16:57:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:48Z" name: success-200-isvc-3eac5-predictor.18a9596994ef3f05 namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: 103939e7-92fc-4dec-a668-f9439d02f241 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18804" uid: d693ccd9-e78c-4c17-abba-e9af253e517c kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:03Z" name: success-200-isvc-3eac5-predictor.18a95974121af8bd namespace: kserve-ci-e2e-test resourceVersion: "19383" uid: dcdf0fb5-9ebb-41ed-8f68-f77c0b07d0b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3eac5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18804" uid: d693ccd9-e78c-4c17-abba-e9af253e517c kind: Event lastTimestamp: "2026-04-24T16:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:03Z" name: success-200-isvc-3eac5-predictor.18a95974128e486a namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: 222539e9-32a6-4f00-b9a2-1a43d49fa2d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: e54893dd-cf33-4bb0-ba62-63dccd458b2c kind: Event lastTimestamp: "2026-04-24T16:57:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:51Z" name: success-200-isvc-3eac5.18a9596614e3f45f namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: ee0022a1-3f48-431b-ba71-f66eeefa37f9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: e54893dd-cf33-4bb0-ba62-63dccd458b2c kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-3eac5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3eac5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-3eac5.18a95966157d49f7 namespace: kserve-ci-e2e-test resourceVersion: "18562" uid: 37eb496a-764e-4b0e-9ad3-10f730e0f88f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: e54893dd-cf33-4bb0-ba62-63dccd458b2c kind: Event lastTimestamp: "2026-04-24T16:57:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3eac5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:02Z" name: success-200-isvc-3eac5.18a95966157d832a namespace: kserve-ci-e2e-test resourceVersion: "18563" uid: fff86573-b635-44b0-83b0-47d62162693b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3eac5 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: e54893dd-cf33-4bb0-ba62-63dccd458b2c kind: Event lastTimestamp: "2026-04-24T16:57:51Z" message: InferenceService [success-200-isvc-3eac5] is Ready metadata: creationTimestamp: "2026-04-24T16:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:51Z" name: success-200-isvc-3eac5.18a95971564f6ccf namespace: kserve-ci-e2e-test resourceVersion: "19263" uid: 6826c8a5-8e72-4fa2-b567-508f403db828 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:57:37.511527Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-510ff-predictor-7d689d4d56-kfdtb to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e21222d15 namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 6755e557-6f4d-4606-a39b-29ff58e3c158 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e3a7d79b2 namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: c82f1173-8bc6-4ef3-9ca8-50221bc32c9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e3be8f42c namespace: kserve-ci-e2e-test resourceVersion: "19117" uid: f1bbb4b6-149e-493f-82ea-916d00e1180b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e3fc46bcb namespace: kserve-ci-e2e-test resourceVersion: "19119" uid: 1a15f285-c1ed-428a-a775-3a3d23a18cc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e4047b51f namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 0dc6a099-a17d-4f77-9586-8d17038bf57c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e40628242 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 09edb993-6f91-461b-a8ce-1216733ac219 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e43318c24 namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: 553a02cb-88db-4b14-9bf1-cb5950689cce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:57:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:38Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e43b31382 namespace: kserve-ci-e2e-test resourceVersion: "19124" uid: 9f52efe7-7bbb-4877-a5f8-7e6efb1971e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T16:58:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:15Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a9596e8d64d084 namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 68f33f28-ad3c-4080-a570-742270a4950f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a959ef45e95711 namespace: kserve-ci-e2e-test resourceVersion: "23396" uid: 02ed29c4-a422-4b35-8c56-da2ffbf26a90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 2b34bdd3-4fb5-4a3e-a5f0-5318dbf718a9 kind: Event lastTimestamp: "2026-04-24T17:06:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:06:52Z" name: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb.18a959ef45e9d2b4 namespace: kserve-ci-e2e-test resourceVersion: "23399" uid: b5c2cb2b-0b98-479e-a099-5c77f740326c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-510ff-predictor-7d689d4d56 namespace: kserve-ci-e2e-test resourceVersion: "19045" uid: 20873d3c-7876-4943-8a8c-1b7ca5abe29a kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'Created pod: success-200-isvc-510ff-predictor-7d689d4d56-kfdtb' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff-predictor-7d689d4d56.18a9596e20878fac namespace: kserve-ci-e2e-test resourceVersion: "19059" uid: e2ba3974-2264-4072-9c73-0cf43f1cec09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: bf5c447e-223e-492e-aff8-da21a304222d kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Scaled up replica set success-200-isvc-510ff-predictor-7d689d4d56 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff-predictor.18a9596e1e138687 namespace: kserve-ci-e2e-test resourceVersion: "19049" uid: 94d8e9ae-a389-4b6e-830e-fb058c3caf4d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19044" uid: 8343d535-dfff-4eb6-97fa-7f637c6bb323 kind: Event lastTimestamp: "2026-04-24T16:58:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:07Z" name: success-200-isvc-510ff-predictor.18a959719c9281f4 namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: f32c7148-0a44-45fb-9644-c2c479267d90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19044" uid: 8343d535-dfff-4eb6-97fa-7f637c6bb323 kind: Event lastTimestamp: "2026-04-24T16:58:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:07Z" name: success-200-isvc-510ff-predictor.18a959719d06844e namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: 09221592-c109-42c3-afef-b7c3ea47628b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 8343d535-dfff-4eb6-97fa-7f637c6bb323 kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: success-200-isvc-510ff-predictor.18a959789ab00232 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 9a8c5917-82da-4de3-a562-3c15dbfc6eca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-510ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 8343d535-dfff-4eb6-97fa-7f637c6bb323 kind: Event lastTimestamp: "2026-04-24T16:58:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:37Z" name: success-200-isvc-510ff-predictor.18a959789b3b3e3a namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 382ecf3e-8dfc-43c4-90e8-151658d41596 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 6f4137d5-aaf7-4da3-bbd2-6498758e071c kind: Event lastTimestamp: "2026-04-24T16:58:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:25Z" name: success-200-isvc-510ff.18a9596e1dba6c22 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 6193eee0-716f-4496-8cd8-30c0c8119bbe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 6f4137d5-aaf7-4da3-bbd2-6498758e071c kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-510ff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-510ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff.18a9596e1e1b26ed namespace: kserve-ci-e2e-test resourceVersion: "19050" uid: 6a75a053-46f7-4045-bb67-032843c8f4a6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 6f4137d5-aaf7-4da3-bbd2-6498758e071c kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-510ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-510ff.18a9596e1e1b5b96 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: b5860278-08a7-4c24-95a6-5d0f22c3cbef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-510ff namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 6f4137d5-aaf7-4da3-bbd2-6498758e071c kind: Event lastTimestamp: "2026-04-24T16:58:25Z" message: InferenceService [success-200-isvc-510ff] is Ready metadata: creationTimestamp: "2026-04-24T16:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:58:25Z" name: success-200-isvc-510ff.18a9597948a98217 namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 808db23d-4f72-43fb-af75-48860249bbb7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:58:13.268371Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-77cb4-predictor-79db889498-sjlnx to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a9597674685f14 namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: a3f6f9d3-a599-4eaf-afae-1f0b8f128eaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a959768d5d09e5 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 9f9f6b74-4638-4008-b48c-fd76f289bfb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a959768eccca00 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: a25d2b31-5f4c-4df8-8eba-a2fcbee48d1f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a95976918281fe namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 487b2805-551b-49b7-8680-b7139c0e00e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a9597691fb107a namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: 09ce4bb4-4d80-4af2-a9b8-7831bbeba320 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a95976920fa749 namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: 3551a003-476b-455d-8247-2d2e84dc2630 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a95976956a6a1d namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: 1360242c-7c63-4871-b6e1-83e0e0c9788b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a9597695f22881 namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: 06520730-8015-4fea-af74-6ecd6b551fa8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:58:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T16:58:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:58:51Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a95976f6cdf507 namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 3597f8fb-276b-4fe0-ac3e-4b41c64d6c0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a959f79de1d198 namespace: kserve-ci-e2e-test resourceVersion: "23813" uid: 8226a98d-dc18-43f4-abed-d91af69bb0ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77cb4-predictor-79db889498-sjlnx namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d51023aa-dae5-4673-8760-e8cf52d4cacb kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-77cb4-predictor-79db889498-sjlnx.18a959f79de2f12c namespace: kserve-ci-e2e-test resourceVersion: "23817" uid: a56e4d51-b40f-4639-8f41-655d35166c9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-77cb4-predictor-79db889498 namespace: kserve-ci-e2e-test resourceVersion: "19540" uid: 6e6d0f80-f524-4011-827d-4ab85adb910b kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: 'Created pod: success-200-isvc-77cb4-predictor-79db889498-sjlnx' metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor-79db889498.18a95976739ca0a1 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 553d25a5-2577-424e-8303-afea6ebd8785 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: fc287cd2-9696-4486-bde9-bf8b9f0666c0 kind: Event lastTimestamp: "2026-04-24T16:58:13Z" message: Scaled up replica set success-200-isvc-77cb4-predictor-79db889498 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:13Z" name: success-200-isvc-77cb4-predictor.18a9597670503e27 namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: 3006b404-a88f-4424-9624-1e2b7913a3f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 425af5cd-1761-4d46-88b2-79b5057063ec kind: Event lastTimestamp: "2026-04-24T16:58:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:43Z" name: success-200-isvc-77cb4-predictor.18a95979ef02627c namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: f493bbc0-0f72-45f1-9caf-93a48bf2499f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 425af5cd-1761-4d46-88b2-79b5057063ec kind: Event lastTimestamp: "2026-04-24T16:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:58:43Z" name: success-200-isvc-77cb4-predictor.18a95979ef89bb69 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: d5c223ca-fa17-440e-baee-2a3fed15a21a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 425af5cd-1761-4d46-88b2-79b5057063ec kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: success-200-isvc-77cb4-predictor.18a95980edb0ae7c namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: cf26a4dc-0736-47a5-b7fb-b4bd901137cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T16:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77cb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 425af5cd-1761-4d46-88b2-79b5057063ec kind: Event lastTimestamp: "2026-04-24T16:59:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:59:28Z" name: success-200-isvc-77cb4-predictor.18a95980ee24a2d0 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: acd8e3f0-ccc8-4c72-aedb-8866ceb2a099 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: 15ccee3c-7057-4971-85ee-f0632338c132 kind: Event lastTimestamp: "2026-04-24T16:59:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:59:01Z" name: success-200-isvc-77cb4.18a959766ff20f48 namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 86c482e3-27b7-4449-92df-0ebb14d6a609 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:59:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77cb4 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 15ccee3c-7057-4971-85ee-f0632338c132 kind: Event lastTimestamp: "2026-04-24T16:59:01Z" message: InferenceService [success-200-isvc-77cb4] is Ready metadata: creationTimestamp: "2026-04-24T16:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:59:01Z" name: success-200-isvc-77cb4.18a95981b27981de namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 4f282a14-308a-4b19-8a8f-4539408827ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:07:28.068244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23829" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8edb5-predictor-7666dfcc56-g6679 to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7a10e9655 namespace: kserve-ci-e2e-test resourceVersion: "23839" uid: cd0237c2-860f-4797-a753-d6c97acaf6a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23846" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7ba65814e namespace: kserve-ci-e2e-test resourceVersion: "23884" uid: fe46d3ae-0d45-49d0-9ba0-b0e807087c6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7bb9a3766 namespace: kserve-ci-e2e-test resourceVersion: "23886" uid: daa8d90c-f78a-43be-89a4-17aafe179443 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7bf095ea7 namespace: kserve-ci-e2e-test resourceVersion: "23888" uid: a7eda512-b934-41b5-b40f-0b2a3059ac63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7bfcf770b namespace: kserve-ci-e2e-test resourceVersion: "23891" uid: fc83ed97-7ed5-4020-8dfb-4897f04006c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7bfde67b6 namespace: kserve-ci-e2e-test resourceVersion: "23892" uid: a5edd707-80ec-489a-92d8-33e0172bcbc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7c397e4c7 namespace: kserve-ci-e2e-test resourceVersion: "23896" uid: 5b1d5d97-75e2-4563-bef1-e3e864c2838a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7c40e9cd1 namespace: kserve-ci-e2e-test resourceVersion: "23897" uid: 8d352a85-b1d3-4f48-9399-bad5bede5b8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:08:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:05Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a959f7e32af9be namespace: kserve-ci-e2e-test resourceVersion: "24408" uid: 9758365a-60da-4a2c-8a13-96a7cbfb4bd0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:24:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:24:47Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a95ae99c79b91e namespace: kserve-ci-e2e-test resourceVersion: "31881" uid: 90b750db-4754-429f-a962-64ee20618e25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679 namespace: kserve-ci-e2e-test resourceVersion: "23833" uid: 8b25dd1a-05c9-4bc1-9002-cc05dbe16ab0 kind: Event lastTimestamp: "2026-04-24T17:24:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:24:47Z" name: success-200-isvc-8edb5-predictor-7666dfcc56-g6679.18a95ae99c7a43a6 namespace: kserve-ci-e2e-test resourceVersion: "31882" uid: 1c732438-fd8a-4f91-a6b3-bdac4bbeb66b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8edb5-predictor-7666dfcc56 namespace: kserve-ci-e2e-test resourceVersion: "23815" uid: b311d12a-47c6-4239-9686-317a9402734e kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Created pod: success-200-isvc-8edb5-predictor-7666dfcc56-g6679' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor-7666dfcc56.18a959f7a078c6ad namespace: kserve-ci-e2e-test resourceVersion: "23834" uid: c1ec7942-8522-4691-a69f-8d1ac78f5246 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: 9843a54f-a411-4cb8-8894-b28531a1c4bd kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: Scaled up replica set success-200-isvc-8edb5-predictor-7666dfcc56 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5-predictor.18a959f79e7e983b namespace: kserve-ci-e2e-test resourceVersion: "23818" uid: 99f72a91-e399-4b1c-baf6-29e707f5cc77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23814" uid: 1b2fc0c9-58a6-4181-83eb-e43a834e6471 kind: Event lastTimestamp: "2026-04-24T17:07:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:58Z" name: success-200-isvc-8edb5-predictor.18a959fb1d00fe40 namespace: kserve-ci-e2e-test resourceVersion: "24208" uid: 7cbbf5db-7ab5-405c-b24f-2bd71b86de4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23814" uid: 1b2fc0c9-58a6-4181-83eb-e43a834e6471 kind: Event lastTimestamp: "2026-04-24T17:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:07:58Z" name: success-200-isvc-8edb5-predictor.18a959fb1d7022de namespace: kserve-ci-e2e-test resourceVersion: "24209" uid: 4b6b7b52-a61e-4845-b088-45017cbf2957 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "24053" uid: 1b2fc0c9-58a6-4181-83eb-e43a834e6471 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: success-200-isvc-8edb5-predictor.18a95a021b5b0a59 namespace: kserve-ci-e2e-test resourceVersion: "24775" uid: 4b58cc7d-bc29-4ff6-a523-b27ca25b1192 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8edb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "24053" uid: 1b2fc0c9-58a6-4181-83eb-e43a834e6471 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: success-200-isvc-8edb5-predictor.18a95a021bc8c352 namespace: kserve-ci-e2e-test resourceVersion: "24776" uid: f8addbc1-19b8-4899-9779-791a7af87b34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23790" uid: 4dea0051-6854-4fee-9c38-cb474492bb8e kind: Event lastTimestamp: "2026-04-24T17:08:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:15Z" name: success-200-isvc-8edb5.18a959f79e5a31d3 namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: b3bf4444-e681-4dd8-909a-2814d33f445a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23790" uid: 4dea0051-6854-4fee-9c38-cb474492bb8e kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-8edb5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8edb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5.18a959f79f08b3d4 namespace: kserve-ci-e2e-test resourceVersion: "23820" uid: 6f6f8c1e-9929-4710-ac33-d58cd185c326 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "23790" uid: 4dea0051-6854-4fee-9c38-cb474492bb8e kind: Event lastTimestamp: "2026-04-24T17:07:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8edb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:07:28Z" name: success-200-isvc-8edb5.18a959f79f08e669 namespace: kserve-ci-e2e-test resourceVersion: "23822" uid: 80d0c0b7-b4b8-4036-b1ef-a7ad5de835ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24507" uid: 4dea0051-6854-4fee-9c38-cb474492bb8e kind: Event lastTimestamp: "2026-04-24T17:08:15Z" message: InferenceService [success-200-isvc-8edb5] is Ready metadata: creationTimestamp: "2026-04-24T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:15Z" name: success-200-isvc-8edb5.18a95a029e23ea5f namespace: kserve-ci-e2e-test resourceVersion: "24508" uid: 2f243e28-850d-4de3-81d9-ac2e7e354d16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:08:02.505293Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9c25c-predictor-69b79585f6-6nvqj to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffa5aa3323 namespace: kserve-ci-e2e-test resourceVersion: "24313" uid: c26fa107-ea14-42b8-b528-9b9887d6fe9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9c25c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffb3fda84e namespace: kserve-ci-e2e-test resourceVersion: "24354" uid: 0c880c5c-cf7d-4118-9b37-30947cc10617 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffe35726c6 namespace: kserve-ci-e2e-test resourceVersion: "24369" uid: 7ab1b9b5-bb12-4b89-b314-a08917ec94dc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffe4a833e1 namespace: kserve-ci-e2e-test resourceVersion: "24371" uid: 4a1d9bde-6e82-46b1-96da-518ce905e705 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffe9922302 namespace: kserve-ci-e2e-test resourceVersion: "24376" uid: c8ee9708-484a-4ef8-84d3-2b8f72cf5016 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffea01e706 namespace: kserve-ci-e2e-test resourceVersion: "24377" uid: f302268a-ecb0-4479-817b-266686d703a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffea0f2ec7 namespace: kserve-ci-e2e-test resourceVersion: "24378" uid: fc364757-90b8-4f2f-a570-48cffee5abbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffed67dba3 namespace: kserve-ci-e2e-test resourceVersion: "24381" uid: 24091501-e3fa-4b55-b9bb-c067e7fe9b3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:08:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:03Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a959ffedd3bfc5 namespace: kserve-ci-e2e-test resourceVersion: "24385" uid: 55bde4c4-6ac8-45f5-9422-c8b72e618a8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T17:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:17:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:20Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a95a0010d2377d namespace: kserve-ci-e2e-test resourceVersion: "28760" uid: 7622df94-d665-4c37-b86a-4aa54ffefb2b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:17:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T17:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:17Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a95a80da1dd72f namespace: kserve-ci-e2e-test resourceVersion: "28727" uid: 25ddce2a-e324-42b3-bfae-73dd4092e86a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:17:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T17:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:17Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a95a80da1e451d namespace: kserve-ci-e2e-test resourceVersion: "28729" uid: 1906ae20-856c-49d3-becd-dabfaa38d5e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: f42d6fa9-8066-419f-8f4f-c795d797031b kind: Event lastTimestamp: "2026-04-24T17:17:20Z" message: 'Readiness probe failed: Get "https://10.133.0.47:8643/healthz": dial tcp 10.133.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T17:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:17:20Z" name: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj.18a95a818542824c namespace: kserve-ci-e2e-test resourceVersion: "28759" uid: 5754efe2-968f-4793-8624-9f8a8c07e51b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9c25c-predictor-69b79585f6 namespace: kserve-ci-e2e-test resourceVersion: "24293" uid: 112d19db-5d24-4e28-85a7-0db62d2897e9 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'Created pod: success-200-isvc-9c25c-predictor-69b79585f6-6nvqj' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c-predictor-69b79585f6.18a959ffa516ace8 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: d5e0168f-e131-408a-ae37-f2e1e2770613 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: fdb6fed1-7034-47e6-bc3d-9219b8a42654 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: Scaled up replica set success-200-isvc-9c25c-predictor-69b79585f6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c-predictor.18a959ffa2f3084f namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 86be8d89-a11f-46ad-a777-0cc3b7d4e4bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24292" uid: 9cc52414-cf4b-4aa8-ad85-4fc8bb96bcbb kind: Event lastTimestamp: "2026-04-24T17:08:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:32Z" name: success-200-isvc-9c25c-predictor.18a95a03218e206a namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: d0f11a0f-5b0d-407c-805a-decdf6d6fac8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T17:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24292" uid: 9cc52414-cf4b-4aa8-ad85-4fc8bb96bcbb kind: Event lastTimestamp: "2026-04-24T17:08:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:32Z" name: success-200-isvc-9c25c-predictor.18a95a0321daa567 namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 141e1fbb-7ebf-488a-be40-099d34bfe8ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 9cc52414-cf4b-4aa8-ad85-4fc8bb96bcbb kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: success-200-isvc-9c25c-predictor.18a95a0a1f4d8b8a namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: f170cbc0-10e3-4181-b1f5-2352e8696d52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T17:08:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c25c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 9cc52414-cf4b-4aa8-ad85-4fc8bb96bcbb kind: Event lastTimestamp: "2026-04-24T17:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:09:17Z" name: success-200-isvc-9c25c-predictor.18a95a0a1fb5516a namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: 7eed9ead-a990-449f-9bfc-93606713fb49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24271" uid: 942a7dc2-fc63-48db-bd87-d9d0180baf30 kind: Event lastTimestamp: "2026-04-24T17:08:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:50Z" name: success-200-isvc-9c25c.18a959ffa2c6b49e namespace: kserve-ci-e2e-test resourceVersion: "24844" uid: cbf15df3-e2af-4261-9e39-e5c8d556e6cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24271" uid: 942a7dc2-fc63-48db-bd87-d9d0180baf30 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-9c25c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c.18a959ffa3245f09 namespace: kserve-ci-e2e-test resourceVersion: "24298" uid: 60abb48b-da56-42a4-a598-eeb4ac130d8d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24271" uid: 942a7dc2-fc63-48db-bd87-d9d0180baf30 kind: Event lastTimestamp: "2026-04-24T17:08:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c25c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T17:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:02Z" name: success-200-isvc-9c25c.18a959ffa324afcc namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: ea44c93f-19a0-419a-8dcf-d4d08ac06e66 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c25c namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: 942a7dc2-fc63-48db-bd87-d9d0180baf30 kind: Event lastTimestamp: "2026-04-24T17:08:50Z" message: InferenceService [success-200-isvc-9c25c] is Ready metadata: creationTimestamp: "2026-04-24T17:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:50Z" name: success-200-isvc-9c25c.18a95a0acb824c37 namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: cd0ca232-11cf-46da-a900-d17c24f7bf53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:56:27.579457Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18174" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f6171-predictor-778d4b8f54-pj5fj to ip-10-0-128-44.ec2.internal metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:56:27Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595dd8db77c9 namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: 861342a4-ae72-4acb-a6f0-38ddab658ff9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df220ad0c namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: 57ef643a-c2be-442c-8e3c-c8184fce5978 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1444" already present on machine metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df42b4623 namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: 53695092-d61d-44c4-af18-c68eac92d7fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df7259397 namespace: kserve-ci-e2e-test resourceVersion: "18226" uid: 23e004da-3d8e-4684-9517-96f497307627 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df795f8e4 namespace: kserve-ci-e2e-test resourceVersion: "18227" uid: 3d158542-f209-4f7c-8641-a63b38aa9d5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df7d111ee namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: d4704318-c6d3-4062-a549-2b9e1a7d7c5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595df9efd6b7 namespace: kserve-ci-e2e-test resourceVersion: "18232" uid: 15aeecf7-189f-495f-a6f5-4f09b7019235 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:56:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:28Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595dfa5f1174 namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: 07faa6c6-5e84-4349-ac1b-ad00c29e94c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:57:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:05Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9595e317d5345 namespace: kserve-ci-e2e-test resourceVersion: "18664" uid: 4b4a8e41-0d69-4b4c-b149-c53cee82c275 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9596e1e9d5cac namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: e133b8b5-c376-4740-b5fe-6f561420c777 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:57:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T16:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:37Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9596e1ea25c7a namespace: kserve-ci-e2e-test resourceVersion: "19053" uid: 26457441-7177-4c55-9791-2e73410950a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: ba06523a-200e-4f01-8d61-6e865e8fb74f kind: Event lastTimestamp: "2026-04-24T16:57:40Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T16:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:57:40Z" name: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj.18a9596eb9ab6b8d namespace: kserve-ci-e2e-test resourceVersion: "19159" uid: c45ee1ad-c5a9-45f3-b7ff-feafebe790db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-44.ec2.internal source: component: kubelet host: ip-10-0-128-44.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f6171-predictor-778d4b8f54 namespace: kserve-ci-e2e-test resourceVersion: "18146" uid: 146cfff8-5a72-4523-8a9e-103700464462 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: 'Created pod: success-200-isvc-f6171-predictor-778d4b8f54-pj5fj' metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:27Z" name: success-200-isvc-f6171-predictor-778d4b8f54.18a9595dd6c75942 namespace: kserve-ci-e2e-test resourceVersion: "18175" uid: e2bb4274-6d7d-472e-97eb-51e1235e4d56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18136" uid: 46d39306-cd4b-43af-bd2f-e02a0537346d kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Scaled up replica set success-200-isvc-f6171-predictor-778d4b8f54 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:27Z" name: success-200-isvc-f6171-predictor.18a9595dc45b618a namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: 44110129-0cb6-4405-82ea-5a8d6068a0bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: bcdaa7e9-5ff8-46c4-8df8-fd5d441a8c6a kind: Event lastTimestamp: "2026-04-24T16:56:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:57Z" name: success-200-isvc-f6171-predictor.18a959614210c4a1 namespace: kserve-ci-e2e-test resourceVersion: "18472" uid: 6e0a6a47-98e9-4747-83fe-c2e2f9a9f438 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: bcdaa7e9-5ff8-46c4-8df8-fd5d441a8c6a kind: Event lastTimestamp: "2026-04-24T16:56:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:56:57Z" name: success-200-isvc-f6171-predictor.18a95961428ee6f0 namespace: kserve-ci-e2e-test resourceVersion: "18473" uid: 6fc6b4c4-2ea6-407e-90bd-727b60520fac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: bcdaa7e9-5ff8-46c4-8df8-fd5d441a8c6a kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: success-200-isvc-f6171-predictor.18a9596841110dd7 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: 72946366-bcf8-4edb-8cd1-fede8db8aab7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6171-predictor namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: bcdaa7e9-5ff8-46c4-8df8-fd5d441a8c6a kind: Event lastTimestamp: "2026-04-24T16:57:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T16:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:57:27Z" name: success-200-isvc-f6171-predictor.18a959684164e331 namespace: kserve-ci-e2e-test resourceVersion: "18900" uid: e4087bcd-14da-41ed-a00d-2c02091db4a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18124" uid: a118562f-c9e5-4c82-a3c9-93fb04e7f671 kind: Event lastTimestamp: "2026-04-24T16:57:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:15Z" name: success-200-isvc-f6171.18a9595dc341c4ad namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: bb1ae676-2d42-407d-b6dc-8db05f890136 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:57:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6171 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: a118562f-c9e5-4c82-a3c9-93fb04e7f671 kind: Event lastTimestamp: "2026-04-24T16:57:15Z" message: InferenceService [success-200-isvc-f6171] is Ready metadata: creationTimestamp: "2026-04-24T16:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:57:15Z" name: success-200-isvc-f6171.18a95968f00afb65 namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: f1149e55-146b-4f1e-bbc1-f77731a93956 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:48:12.236965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-2c4dd-5f99fd554-5kkch to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a958ea8424f2de namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 82053d81-d54a-45c4-ad0c-4e397a8cfe37 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a958ea9d19b77a namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: d1a1ed14-5a70-49e7-a788-6e41fb038fab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2c4dd} kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a958ea9eb2ed12 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 1e00d52e-20e5-44ad-8fa4-75a5ae265b4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2c4dd} kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: 'Created container: switch-graph-2c4dd' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a958eaa1c23977 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: feede856-eba6-4180-bd43-6356ae1bca64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2c4dd} kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: Started container switch-graph-2c4dd metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a958eaa237796d namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 424ef9bc-8738-405b-9df1-fd85861b5b67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2c4dd} kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:56:27Z" message: Stopping container switch-graph-2c4dd metadata: creationTimestamp: "2026-04-24T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:27Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a9595db8c475e2 namespace: kserve-ci-e2e-test resourceVersion: "18122" uid: a91737bb-cb81-44be-bf5e-e77b0ffd2a1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2c4dd} kind: Pod name: switch-graph-2c4dd-5f99fd554-5kkch namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b0aacbe3-2a72-4fbd-ac66-8ba97f59b1d7 kind: Event lastTimestamp: "2026-04-24T16:56:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:56:54Z" name: switch-graph-2c4dd-5f99fd554-5kkch.18a9595e3123c73a namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 877129c6-98da-49c8-9467-d5ca67d08efa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-2c4dd-5f99fd554 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: cef012a6-eb37-4411-8f2e-4a4e35270adc kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: 'Created pod: switch-graph-2c4dd-5f99fd554-5kkch' metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd-5f99fd554.18a958ea82e68dba namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: 01df4b14-c7d6-433b-b059-4bf8e985e300 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 9217c36c-dff9-4fa9-902f-a96603b8b581 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: Scaled up replica set switch-graph-2c4dd-5f99fd554 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd.18a958ea806eb645 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 7157db8c-634e-424d-a140-0c17f674efa0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:48:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: bf7bc69e-2e96-4ae1-91af-dedcc1e6ebd5 kind: Event lastTimestamp: "2026-04-24T16:48:12Z" message: InferenceGraph [switch-graph-2c4dd] is Ready metadata: creationTimestamp: "2026-04-24T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:48:12Z" name: switch-graph-2c4dd.18a958ea87860dfa namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: f5fec86b-0503-4ed7-ad82-67d3aca2fb38 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: b6d08e8f-7b29-43f8-987d-f6d5de3b5223 kind: Event lastTimestamp: "2026-04-24T16:48:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:42Z" name: switch-graph-2c4dd.18a958edffb93a83 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 3b64db34-ad45-4904-8a76-7b7acdfcdd6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T16:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2c4dd namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: b6d08e8f-7b29-43f8-987d-f6d5de3b5223 kind: Event lastTimestamp: "2026-04-24T16:48:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:48:42Z" name: switch-graph-2c4dd.18a958ee0038d35c namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 0c8f6364-f8ef-489e-80fd-654a2f4ebe89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T16:46:57.721243Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13564" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-32209-6559f64bb7-gspth to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T16:46:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T16:46:57Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d92aa931c2 namespace: kserve-ci-e2e-test resourceVersion: "13569" uid: eef73f21-6e90-4b6a-b0bb-d17b3f114e06 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:46:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-32209-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T16:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:57Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d9332e6019 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: 5947f8c5-7a18-4dac-ba10-8ff20297e548 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:58Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:46:58Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T16:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T16:46:58Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d967a42904 namespace: kserve-ci-e2e-test resourceVersion: "13587" uid: 5632ed30-d7c1-4dde-acbe-9e29f5d8e220 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:46:58Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" metadata: creationTimestamp: "2026-04-24T16:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:46:58Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d96915572a namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: c3b5079b-1e51-43bf-a0b2-9a4a65a4b7d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:47:01Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" in 2.289s (2.289s including waiting). Image size: 216346204 bytes.' metadata: creationTimestamp: "2026-04-24T16:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:01Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d9f1868fed namespace: kserve-ci-e2e-test resourceVersion: "13608" uid: f00337ac-0b88-418c-9de8-b070489590f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:47:01Z" message: 'Created container: switch-graph-32209' metadata: creationTimestamp: "2026-04-24T16:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:01Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d9f4a5a37f namespace: kserve-ci-e2e-test resourceVersion: "13609" uid: a8f08336-2375-4af2-ab44-cb6037309f59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:47:01Z" message: Started container switch-graph-32209 metadata: creationTimestamp: "2026-04-24T16:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:01Z" name: switch-graph-32209-6559f64bb7-gspth.18a958d9f5107f86 namespace: kserve-ci-e2e-test resourceVersion: "13610" uid: 8acbc93f-b5b3-48a5-8105-e4fd90fac4f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:47:11Z" message: Stopping container switch-graph-32209 metadata: creationTimestamp: "2026-04-24T16:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:11Z" name: switch-graph-32209-6559f64bb7-gspth.18a958dc795590ab namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: eb57a447-bfd0-422f-a9bf-a95acfa11f42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T16:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32209} kind: Pod name: switch-graph-32209-6559f64bb7-gspth namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 09fa39fb-000b-4fe7-9efa-229106525764 kind: Event lastTimestamp: "2026-04-24T16:47:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T16:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T16:47:37Z" name: switch-graph-32209-6559f64bb7-gspth.18a958dcaff08ec5 namespace: kserve-ci-e2e-test resourceVersion: "14104" uid: 5e9c164b-08a2-4ed6-94e8-c1573bb14cd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-32209-6559f64bb7 namespace: kserve-ci-e2e-test resourceVersion: "13560" uid: d0c1cd84-dce2-4776-922c-2c731e814b36 kind: Event lastTimestamp: "2026-04-24T16:46:57Z" message: 'Created pod: switch-graph-32209-6559f64bb7-gspth' metadata: creationTimestamp: "2026-04-24T16:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:57Z" name: switch-graph-32209-6559f64bb7.18a958d92a26ef72 namespace: kserve-ci-e2e-test resourceVersion: "13568" uid: c4802791-8ba6-4206-9c85-8e74e5931006 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-32209 namespace: kserve-ci-e2e-test resourceVersion: "13559" uid: bb6623f1-fb13-4c3e-a79f-f7a4715cc3f2 kind: Event lastTimestamp: "2026-04-24T16:46:57Z" message: Scaled up replica set switch-graph-32209-6559f64bb7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T16:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T16:46:57Z" name: switch-graph-32209.18a958d9291dcf91 namespace: kserve-ci-e2e-test resourceVersion: "13562" uid: 10b83db0-fb1b-434a-8735-90182910afc4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T16:46:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32209 namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: f3136941-d9ca-4b8e-ae37-f3cbd847a856 kind: Event lastTimestamp: "2026-04-24T16:46:57Z" message: InferenceGraph [switch-graph-32209] is Ready metadata: creationTimestamp: "2026-04-24T16:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T16:46:57Z" name: switch-graph-32209.18a958d92f404d18 namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: a2b4940b-3e53-4959-b81c-2ac6daf575ab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T17:08:28.120176Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8edb5-5fc995bf6d-m66br to ip-10-0-131-47.ec2.internal metadata: creationTimestamp: "2026-04-24T17:08:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T17:08:28Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a059c6e4851 namespace: kserve-ci-e2e-test resourceVersion: "24636" uid: 4b062a7f-eec6-443b-8742-05a33569211e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57b778cfc4-hdcg4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:08:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-8edb5-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:28Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a05a6ed34f5 namespace: kserve-ci-e2e-test resourceVersion: "24649" uid: ce374d56-26fc-433b-ac18-f6684236e032 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:08:29Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T17:08:29Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a05d94c431a namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: 43be87bf-bc26-491c-92fa-b9b3459aee02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8edb5} kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:08:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9080f98c95e0f799bceecaf3d1b297de483651b395e63f9d105b355570ba5858" already present on machine metadata: creationTimestamp: "2026-04-24T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:29Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a05daa106b5 namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: 6f724cc9-0e37-4c55-bcda-58b50f707bad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8edb5} kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:08:29Z" message: 'Created container: switch-graph-8edb5' metadata: creationTimestamp: "2026-04-24T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:29Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a05dd8d4994 namespace: kserve-ci-e2e-test resourceVersion: "24662" uid: 4e610617-45ba-4c28-80ce-510de128d8a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8edb5} kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:08:29Z" message: Started container switch-graph-8edb5 metadata: creationTimestamp: "2026-04-24T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:08:29Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95a05de061070 namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: 2aaacb5d-f763-45ca-8091-bb53afac00d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8edb5} kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:24:47Z" message: Stopping container switch-graph-8edb5 metadata: creationTimestamp: "2026-04-24T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:24:47Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95ae9953e2bf7 namespace: kserve-ci-e2e-test resourceVersion: "31867" uid: 25b9d6d4-1901-4da7-b41a-227fd504c71a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T17:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8edb5} kind: Pod name: switch-graph-8edb5-5fc995bf6d-m66br namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: db54f995-a27f-4f3e-97da-f93dbc9c5e0b kind: Event lastTimestamp: "2026-04-24T17:25:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T17:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T17:25:10Z" name: switch-graph-8edb5-5fc995bf6d-m66br.18a95aea5a46a4cf namespace: kserve-ci-e2e-test resourceVersion: "32108" uid: 4c7fdf8a-c394-447c-82b7-18e4e18d0225 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-47.ec2.internal source: component: kubelet host: ip-10-0-131-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8edb5-5fc995bf6d namespace: kserve-ci-e2e-test resourceVersion: "24626" uid: 338b994b-8135-4ba5-852b-968c58a678c3 kind: Event lastTimestamp: "2026-04-24T17:08:28Z" message: 'Created pod: switch-graph-8edb5-5fc995bf6d-m66br' metadata: creationTimestamp: "2026-04-24T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:28Z" name: switch-graph-8edb5-5fc995bf6d.18a95a059be9e845 namespace: kserve-ci-e2e-test resourceVersion: "24632" uid: 5aa4150e-dda9-4d28-a61f-0a0f4fefa05c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24624" uid: c46bad5d-4338-47a4-b2ef-ee748fb25c16 kind: Event lastTimestamp: "2026-04-24T17:08:28Z" message: Scaled up replica set switch-graph-8edb5-5fc995bf6d from 0 to 1 metadata: creationTimestamp: "2026-04-24T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:28Z" name: switch-graph-8edb5.18a95a059b4196d0 namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: b81c1a3a-7156-44c4-94cc-63daa4c40cf7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: f3959913-bb55-4337-94ca-e057c6831abf kind: Event lastTimestamp: "2026-04-24T17:08:28Z" message: InferenceGraph [switch-graph-8edb5] is Ready metadata: creationTimestamp: "2026-04-24T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T17:08:28Z" name: switch-graph-8edb5.18a95a059fdd63f1 namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 3079d738-fb6b-4672-b086-9b13e00c807c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24625" uid: 3839cf99-8be2-4990-846f-226353954886 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: switch-graph-8edb5.18a95a0919d0339a namespace: kserve-ci-e2e-test resourceVersion: "24773" uid: 89a75075-4587-40b9-bfa8-5ca1d4da7846 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24625" uid: 3839cf99-8be2-4990-846f-226353954886 kind: Event lastTimestamp: "2026-04-24T17:08:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:43Z" name: switch-graph-8edb5.18a95a091a3ab291 namespace: kserve-ci-e2e-test resourceVersion: "24777" uid: 24d4b728-0783-4018-a916-0181480bd881 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 3839cf99-8be2-4990-846f-226353954886 kind: Event lastTimestamp: "2026-04-24T17:08:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:58Z" name: switch-graph-8edb5.18a95a0c99101bba namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 58a5e336-e4d5-4f88-a19d-d94698b85abb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T17:08:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8edb5 namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 3839cf99-8be2-4990-846f-226353954886 kind: Event lastTimestamp: "2026-04-24T17:08:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T17:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T17:08:58Z" name: switch-graph-8edb5.18a95a0c99826fad namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: 7bcf32be-5556-4aa3-859c-a17559c86ac7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32119"