--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:59:54.814816Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1175d-68b6c76bfd-vkdb6 to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:59:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:59:54Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f28d30cf7 namespace: kserve-ci-e2e-test resourceVersion: "22255" uid: b7c3273d-00d4-4e77-a480-0dc395e11bb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:55Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T17:59:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-1175d-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:55Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f364adea8 namespace: kserve-ci-e2e-test resourceVersion: "22267" uid: 3ed2602e-d3dd-4486-82b4-bab5e74e62ad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:55Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22259" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T17:59:55Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:59:55Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f6eaa7ba5 namespace: kserve-ci-e2e-test resourceVersion: "22280" uid: 3c5a36e7-6d84-4a6c-83f5-7b14958fc09d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1175d} kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T17:59:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:56Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f70260a7f namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 59ae467b-a3b0-42a2-9e00-0f68ae903a02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1175d} kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T17:59:56Z" message: 'Created container: ensemble-graph-1175d' metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:56Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f7312949b namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 212c3b6e-44a4-4712-b0d8-7f12eb510daa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1175d} kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T17:59:56Z" message: Started container ensemble-graph-1175d metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:56Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62d8f7384532b namespace: kserve-ci-e2e-test resourceVersion: "22284" uid: d2fd2191-d038-4f83-aba2-5371daba0704 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1175d} kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Stopping container ensemble-graph-1175d metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:09Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62e0253b48b2f namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 96f39723-4438-4ba9-a7eb-7920b3ec9eae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1175d} kind: Pod name: ensemble-graph-1175d-68b6c76bfd-vkdb6 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: de3fbad8-f28a-4f54-80ef-8dde100dcbb0 kind: Event lastTimestamp: "2026-09-17T18:08:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:37Z" name: ensemble-graph-1175d-68b6c76bfd-vkdb6.18d62e031fa44b84 namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: b3a606dc-5bcc-499b-8450-dafb171af031 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1175d-68b6c76bfd namespace: kserve-ci-e2e-test resourceVersion: "22245" uid: 66b61abf-c941-4968-8369-dffc5124284f kind: Event lastTimestamp: "2026-09-17T17:59:54Z" message: 'Created pod: ensemble-graph-1175d-68b6c76bfd-vkdb6' metadata: creationTimestamp: "2026-09-17T17:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:54Z" name: ensemble-graph-1175d-68b6c76bfd.18d62d8f28164100 namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: afa61590-0550-4b29-9cf3-ca8e85932e37 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: 3650ca4c-dc2a-4030-b949-7836d1bcf91e kind: Event lastTimestamp: "2026-09-17T17:59:54Z" message: Scaled up replica set ensemble-graph-1175d-68b6c76bfd from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:54Z" name: ensemble-graph-1175d.18d62d8f26ada17b namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 75b0b9b2-7a5f-402c-afb1-f720c7afe65c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22261" uid: af3a20aa-ec7a-4967-a773-14716c1d95f6 kind: Event lastTimestamp: "2026-09-17T17:59:54Z" message: InferenceGraph [ensemble-graph-1175d] is Ready metadata: creationTimestamp: "2026-09-17T17:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:54Z" name: ensemble-graph-1175d.18d62d8f2eea7a74 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: d0c40e70-a1f9-468a-8709-dcc45932b05e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: de765a2d-4406-45fc-90c4-e62399b663b1 kind: Event lastTimestamp: "2026-09-17T18:00:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:09Z" name: ensemble-graph-1175d.18d62d92a59d119b namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 41f468ec-d40a-42b3-bad2-e520c8e38837 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: de765a2d-4406-45fc-90c4-e62399b663b1 kind: Event lastTimestamp: "2026-09-17T18:00:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:09Z" name: ensemble-graph-1175d.18d62d92a5fc3761 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: 4b42cbd0-04a2-4d37-bbca-7cae86f1d951 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: de765a2d-4406-45fc-90c4-e62399b663b1 kind: Event lastTimestamp: "2026-09-17T18:00:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:24Z" name: ensemble-graph-1175d.18d62d9624e11695 namespace: kserve-ci-e2e-test resourceVersion: "22516" uid: 7a82e9f8-6cce-4805-a377-b0ae845d18f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1175d namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: de765a2d-4406-45fc-90c4-e62399b663b1 kind: Event lastTimestamp: "2026-09-17T18:00:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:24Z" name: ensemble-graph-1175d.18d62d96253ce4ac namespace: kserve-ci-e2e-test resourceVersion: "22518" uid: 8d7a42de-f048-465c-872b-69008a58a732 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:30.338611Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1fda1-756f77b956-96xnd to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7b7da62552 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: e2077d1e-cfe1-46b4-8111-31878037e5ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7ba0ea5c41 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 25c57294-2297-4505-a5c8-b13b7e7d1540 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1fda1} kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7ba35abe4d namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 66ffc7eb-a435-4ec0-9908-1a43038d8380 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1fda1} kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created container: ensemble-graph-1fda1' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7ba672de98 namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 2c07c0f3-c90c-44f6-8c27-d2149672a00d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1fda1} kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Started container ensemble-graph-1fda1 metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7ba6ed0533 namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: db63d7ec-3c8c-45fb-854a-d37cfee957b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1fda1} kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Stopping container ensemble-graph-1fda1 metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:44Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7ec9759e11 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: bf91b3cd-4b25-427c-bea4-ef2499fd2e31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1fda1} kind: Pod name: ensemble-graph-1fda1-756f77b956-96xnd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 71f6bdcd-2728-4385-bec1-9f4fc1bcdc36 kind: Event lastTimestamp: "2026-09-17T17:59:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:12Z" name: ensemble-graph-1fda1-756f77b956-96xnd.18d62d7f77a61743 namespace: kserve-ci-e2e-test resourceVersion: "21689" uid: 5f4b39b2-13d8-4fd7-80a0-e9626f385783 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1fda1-756f77b956 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 5b8027e5-9bac-4c3f-b873-3859d6abdca6 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'Created pod: ensemble-graph-1fda1-756f77b956-96xnd' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1-756f77b956.18d62d7b7cbc73d6 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 2e245174-def7-47e2-8962-f48db41f847b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 25664662-a0d1-4cb9-876c-b9ad02796fc4 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Scaled up replica set ensemble-graph-1fda1-756f77b956 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1.18d62d7b7ae8e907 namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: ebeb28bc-8240-4136-8793-ad79278592eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 7ff0c5ab-566d-4593-8b88-b02ae9765ec1 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: InferenceGraph [ensemble-graph-1fda1] is Ready metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1.18d62d7b83a4fd00 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 9cbfafbc-470f-4c0e-b7bf-c7d959f3f802 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 7ff0c5ab-566d-4593-8b88-b02ae9765ec1 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-1fda1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1.18d62d7b88a27c9e namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: b12e52c7-07dd-44cb-acb6-074639441ffb reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 7ff0c5ab-566d-4593-8b88-b02ae9765ec1 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:30Z" name: ensemble-graph-1fda1.18d62d7b88a2c3ec namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 2e7f5a09-08ed-4f81-a2b9-e32de6ac94f9 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:45.049855Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1175d-predictor-6784b69858-d8x9r to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7eea81e358 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 717d77a2-d346-482c-8969-15506012fc71 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f0c9fd3fa namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: f3498141-ad28-43cc-a493-7fb910255fda reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f0e66a391 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 0285c504-d45c-43ff-8fc0-fc6192eaf04d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f1152ab54 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 6759c9ec-6732-45f4-9c88-3e9f77b99ff5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f11c3aca7 namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: dd3cbb14-5e3c-4f00-8a08-2039a950f9d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f11cce0bc namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 499d51d5-59aa-4b03-ad19-c775116ca62c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f1551e5b4 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 92b53a37-0bf6-4110-9d58-160945c3537c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f15c6e684 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: f82ea2bf-e6ca-41bf-9792-6733e702ec89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T18:08:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:12Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62d7f4120506a namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: a2c97442-60ac-4974-b334-0972786f7e57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:09Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62e0267d1cdaf namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 1806fbdc-94db-4802-8b89-52fdb4409460 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:09Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62e0267d2b8e2 namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: 3c6e5f4b-94c8-4e84-8ed2-1a2b8d1edefc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1175d-predictor-6784b69858-d8x9r namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 53d6cea0-bed8-4b87-a22e-a770be732246 kind: Event lastTimestamp: "2026-09-17T18:08:12Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:12Z" name: error-404-isvc-1175d-predictor-6784b69858-d8x9r.18d62e03099a3174 namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: 577a2b15-c4aa-4102-844a-92ba47ae1819 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1175d-predictor-6784b69858 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 81ad36e8-4cdc-47b4-87ee-18682c574843 kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: 'Created pod: error-404-isvc-1175d-predictor-6784b69858-d8x9r' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:45Z" name: error-404-isvc-1175d-predictor-6784b69858.18d62d7ee9d175c4 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 797f811a-beec-4197-8d72-5490d22a2a12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 7a73c46e-6ecc-4e7a-ab47-423be5447aae kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Scaled up replica set error-404-isvc-1175d-predictor-6784b69858 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:44Z" name: error-404-isvc-1175d-predictor.18d62d7ee72ed9bf namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 3621fa14-9a9e-4dd3-978a-3d20bfb46fe3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 537ca588-7dd3-4059-bdd4-65f460135679 kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: error-404-isvc-1175d-predictor.18d62d8265f97c8d namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: 40b45d33-7b7d-42a1-bcf0-57f1687dc443 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 537ca588-7dd3-4059-bdd4-65f460135679 kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: error-404-isvc-1175d-predictor.18d62d82668f28f1 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: dbce578f-2e9f-401e-bb71-82f41abd5514 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 537ca588-7dd3-4059-bdd4-65f460135679 kind: Event lastTimestamp: "2026-09-17T18:00:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:00Z" name: error-404-isvc-1175d-predictor.18d62d8ce36bdc38 namespace: kserve-ci-e2e-test resourceVersion: "22324" uid: fd060d94-f6ff-4181-a2e9-ba3185c536e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 537ca588-7dd3-4059-bdd4-65f460135679 kind: Event lastTimestamp: "2026-09-17T18:00:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:00Z" name: error-404-isvc-1175d-predictor.18d62d8ce40bad3d namespace: kserve-ci-e2e-test resourceVersion: "22326" uid: 085d744d-a276-4c98-87b2-8a36d739f456 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1175d namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: c0ed73e7-be00-41a9-82a2-2aa2041c06d4 kind: Event lastTimestamp: "2026-09-17T17:59:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:32Z" name: error-404-isvc-1175d.18d62d7ee742b6e9 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: a5c40a2d-0aea-4269-9284-d3bdf648b84f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1175d namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: c0ed73e7-be00-41a9-82a2-2aa2041c06d4 kind: Event lastTimestamp: "2026-09-17T17:59:32Z" message: InferenceService [error-404-isvc-1175d] is Ready metadata: creationTimestamp: "2026-09-17T17:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:32Z" name: error-404-isvc-1175d.18d62d89fe2de20c namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 50c13736-7c69-4c34-ac21-44bbd0d5e0ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:59:41.255777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c00a434aa namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 29e52646-2202-46f6-bc62-2d376e26824c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1e4e7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c0a662521 namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: b2e24876-ffc5-4c2d-9d8f-b42ec5d19e27 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c467620d1 namespace: kserve-ci-e2e-test resourceVersion: "22119" uid: 8c606ccb-1b3b-4deb-952e-9f059b87bf15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4837987e namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 454c1396-0218-4ec1-8b13-3439c9ff93d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4b530c8c namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: c5b6f351-a600-4553-bf75-ae385d59cc08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4bc97ea5 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: e6618695-d7f3-4b9b-900e-74c52ea17fd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4bd443c5 namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 12eb7066-b598-426c-a1b6-af1e6e258915 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4ed6d22f namespace: kserve-ci-e2e-test resourceVersion: "22125" uid: a2acf8d0-b221-4442-ab33-7e7d4229115e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T17:59:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:42Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c4f471834 namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: fb69b032-c6a1-469e-ab24-2c9cfbea1f80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T18:08:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:58Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62d8c5c5bd4d4 namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: a89392fb-973b-4ac0-a0c6-b787621fb0b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62e0d2e3687a6 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 45abf8f3-7995-4d72-92d6-f654d137ad2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62e0d2e370cb8 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 714f6795-9cca-4c4d-8b0a-48f3ffc9b4c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 7637bb8f-4cf3-4980-8031-8c3862060f3b kind: Event lastTimestamp: "2026-09-17T18:08:58Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:58Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn.18d62e0dd0c7a06f namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: 630aaf15-6710-4555-8f69-ea2b4d2b6f69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1e4e7-predictor-86f4b8b95d namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 7a1fd955-d5cf-4cb7-9ca8-9a685a7f0478 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: 'Created pod: error-404-isvc-1e4e7-predictor-86f4b8b95d-glqzn' metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-1e4e7-predictor-86f4b8b95d.18d62d8c00225478 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: ffb0352f-e799-465a-a62b-59e96afea573 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22069" uid: 56321d77-bbf7-4e1f-b3d8-32ef9601b725 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Scaled up replica set error-404-isvc-1e4e7-predictor-86f4b8b95d from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-1e4e7-predictor.18d62d8bfe2b8d05 namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: 6dcf30e0-841a-456b-91d8-35de2b486218 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 67912952-3c18-4ead-8a6b-a4ba1a37278d kind: Event lastTimestamp: "2026-09-17T18:00:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:26Z" name: error-404-isvc-1e4e7-predictor.18d62d8f7cff0017 namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: a30160a3-4b93-4f91-b115-1af98bdefeac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 67912952-3c18-4ead-8a6b-a4ba1a37278d kind: Event lastTimestamp: "2026-09-17T18:00:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:26Z" name: error-404-isvc-1e4e7-predictor.18d62d8f7d629788 namespace: kserve-ci-e2e-test resourceVersion: "22532" uid: 3919d472-9b40-4edc-8f62-623a027a3616 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 67912952-3c18-4ead-8a6b-a4ba1a37278d kind: Event lastTimestamp: "2026-09-17T18:00:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:56Z" name: error-404-isvc-1e4e7-predictor.18d62d99faabc070 namespace: kserve-ci-e2e-test resourceVersion: "22808" uid: be15457c-1f8b-42e5-8180-391ca28ec184 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 67912952-3c18-4ead-8a6b-a4ba1a37278d kind: Event lastTimestamp: "2026-09-17T18:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:56Z" name: error-404-isvc-1e4e7-predictor.18d62d99fb44b01c namespace: kserve-ci-e2e-test resourceVersion: "22809" uid: 219c1c18-052b-445a-b37e-ee8d004934e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 0734f76c-3948-4475-82c6-177297086d02 kind: Event lastTimestamp: "2026-09-17T18:00:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:00:28Z" name: error-404-isvc-1e4e7.18d62d8bfe2463ab namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 7eb9fc3b-73f1-43c7-98f4-8392759e2000 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 0734f76c-3948-4475-82c6-177297086d02 kind: Event lastTimestamp: "2026-09-17T18:00:28Z" message: InferenceService [error-404-isvc-1e4e7] is Ready metadata: creationTimestamp: "2026-09-17T18:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:00:28Z" name: error-404-isvc-1e4e7.18d62d97194418e9 namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: aebee498-f7a5-4c94-b7f3-cc6f8fcbc67c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:30.375752Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1fda1-predictor-c86ccb888-n4lkk to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6d87958564 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: a727596f-e5a1-49c7-9f45-79f4458488f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1fda1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6d95ea1540 namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: 0ec2c170-9828-4359-a7e6-9ad6222b4d58 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dcd6d9039 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 629e5ac5-7a0a-4097-87d8-7a1980e07c4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dcf26083d namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 02a39cdc-9c86-4e99-9d04-e1ffc535df57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dd203f3a2 namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 428cd77f-1857-4bc4-b2d5-a37f10a08e3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dd2740e2d namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 905c901c-bd23-4715-b9bb-9dd639e512a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dd27d7d99 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 0b630c46-c84c-47ad-ab00-e33a5b00a58c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dd5df7776 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 9a21bf13-ae5f-4b32-899f-23f64e7a38af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6dd64ce181 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 04ce2d12-e33b-4929-bca0-a15c79a8acce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:58:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:08Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d6def88e94c namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 07e30399-cfff-4c07-9b6d-76d00e1e3a06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:44Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d7ed75751f2 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 6a485bd2-7136-44ee-b1ed-d8ba44f0c291 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: e4cba543-f1b7-47ad-b3bc-0d1eac81f333 kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:44Z" name: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk.18d62d7ed757e2fc namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 9c2b7cbb-4160-4a17-9f25-90391c25cba4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1fda1-predictor-c86ccb888 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 3dca8516-011b-45e2-abf7-7d30a2befa15 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created pod: error-404-isvc-1fda1-predictor-c86ccb888-n4lkk' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1-predictor-c86ccb888.18d62d6d86e5eed3 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 40483a37-c643-4899-8db4-4ce3392b56d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 302d70ca-7466-4c4f-bc58-9fcc5d64ac55 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Scaled up replica set error-404-isvc-1fda1-predictor-c86ccb888 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1-predictor.18d62d6d84944d2d namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 360d73ad-ffc9-4600-809a-53c1f5ad0bc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: a0607d57-7b43-4eda-b47a-005d319d2024 kind: Event lastTimestamp: "2026-09-17T17:58:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:15Z" name: error-404-isvc-1fda1-predictor.18d62d71034d004e namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 2f0dd9b2-9fe9-433e-b32b-d4ba4ec6a91e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: a0607d57-7b43-4eda-b47a-005d319d2024 kind: Event lastTimestamp: "2026-09-17T17:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:15Z" name: error-404-isvc-1fda1-predictor.18d62d7103e0b8db namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 21d2ab8c-8beb-4c04-a9f3-40da322cc7b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: a0607d57-7b43-4eda-b47a-005d319d2024 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: error-404-isvc-1fda1-predictor.18d62d7b812886aa namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 19e2a40e-87d9-49d6-8c95-93d1853cb5aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: a0607d57-7b43-4eda-b47a-005d319d2024 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: error-404-isvc-1fda1-predictor.18d62d7b81b71a7b namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 9697260f-e669-4c5f-810f-59c57908205c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 39fd153b-1ea9-40cc-8b79-cccbf1bc903d kind: Event lastTimestamp: "2026-09-17T17:58:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:18Z" name: error-404-isvc-1fda1.18d62d6d84ad5651 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 6d560065-d3d3-46b2-9743-5cf57b3037de reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 39fd153b-1ea9-40cc-8b79-cccbf1bc903d kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-1fda1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1.18d62d6d856ba8f3 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 69b57c4f-0597-4476-8639-1683975ded52 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 39fd153b-1ea9-40cc-8b79-cccbf1bc903d kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-1fda1.18d62d6d856bf115 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 68becec8-1cf6-47e6-9133-52bc7fade962 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 39fd153b-1ea9-40cc-8b79-cccbf1bc903d kind: Event lastTimestamp: "2026-09-17T17:58:18Z" message: InferenceService [error-404-isvc-1fda1] is Ready metadata: creationTimestamp: "2026-09-17T17:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:18Z" name: error-404-isvc-1fda1.18d62d78aef594c1 namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: c8b407b3-f701-488b-a552-15e9864b654b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:45.463141Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4a129-predictor-579fd87984-zc225 to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:45Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cd75fcbfefd namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 170306e1-8d7b-4a15-882d-a189e27fbf12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4a129-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:45Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cd76d987c74 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 425c9a91-418f-4835-82dd-25617a5258f0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:46Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cd7a57fc122 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 061f1c76-0266-4845-b47b-8bec24fd0ddd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cd7a7399e63 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 1c23ae0e-70be-44e3-b8da-97bff88a7ec4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" in 11.594s (11.594s including waiting). Image size: 1355797068 bytes.' metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cda5a4d819c namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 39db0785-8821-4edf-afb3-35153b6e9f2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cda5d9bb076 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: b13b83ff-9eae-43a8-a002-cc580aa07e7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cda5e0b5cef namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 2c6cf194-8582-4ae2-ab06-a0a188735da8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cda5e17d4f3 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 6103c3fe-a1ce-44f0-b2d6-6bca7bdf4278 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:47:00Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.24s (2.24s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:00Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cdae3a4a2ad namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 7480c7ea-1be1-478f-bb26-0a2b7e1262be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:47:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:00Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cdae7a69bf7 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 50b4cbf3-e9c2-4e8b-ac43-54465f246eb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:47:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:00Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cdae829f882 namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: 9606fe1d-ce3e-438e-adbf-16fde3bca4b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cdb044c20fa namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 1f888e62-6404-46a0-a472-36173f7fce04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:15Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cec531d831f namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 50aaea46-9c30-48a0-af7c-e20ed7c2dfa4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:15Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cec531e63ff namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: ac7a4e10-3b73-46d7-9b39-c6d252d22b7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a129-predictor-579fd87984-zc225 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 209300bb-fba8-4b66-95fb-77b9d25418e4 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-4a129-predictor-579fd87984-zc225.18d62cecb67cf812 namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 16c6fbcd-f5fc-49c1-aa13-ffb8c2f5d96d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4a129-predictor-579fd87984 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 39f0fadc-d2f5-4cdc-9d17-f80fd45e0a0f kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created pod: error-404-isvc-4a129-predictor-579fd87984-zc225' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: error-404-isvc-4a129-predictor-579fd87984.18d62cd75f3d4435 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: a034e58e-9553-4eef-8a98-03f412203b08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: b80b3aac-8c11-4acb-a37b-dedfbb84b83a kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Scaled up replica set error-404-isvc-4a129-predictor-579fd87984 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: error-404-isvc-4a129-predictor.18d62cd75ada5f1d namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 5769433d-17d2-48aa-b65d-7c41bd0cdc0c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: ed5a2a87-db92-4ca0-b576-994452b212b8 kind: Event lastTimestamp: "2026-09-17T17:47:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:15Z" name: error-404-isvc-4a129-predictor.18d62cdad98890fd namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4d5f6f56-697c-40f7-9fee-b3ae789ced77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: ed5a2a87-db92-4ca0-b576-994452b212b8 kind: Event lastTimestamp: "2026-09-17T17:47:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:15Z" name: error-404-isvc-4a129-predictor.18d62cdad9e41cac namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: bdf4f7c9-2dad-4951-9f56-ad48d9c6d162 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: ed5a2a87-db92-4ca0-b576-994452b212b8 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: error-404-isvc-4a129-predictor.18d62ce1d86491f7 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: effe678f-c334-49b0-9b04-006a2787c565 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: ed5a2a87-db92-4ca0-b576-994452b212b8 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: error-404-isvc-4a129-predictor.18d62ce1d92e3abe namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: eeb554d2-8cc9-4555-a2f3-27438cbfb943 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: a0cca6d8-1218-4e1f-99f7-d846b7c4819e kind: Event lastTimestamp: "2026-09-17T17:47:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:47Z" name: error-404-isvc-4a129.18d62cd75ae2380b namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 85606d83-4fba-49bd-8423-5bf051dba842 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: a0cca6d8-1218-4e1f-99f7-d846b7c4819e kind: Event lastTimestamp: "2026-09-17T17:47:47Z" message: InferenceService [error-404-isvc-4a129] is Ready metadata: creationTimestamp: "2026-09-17T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:47Z" name: error-404-isvc-4a129.18d62ce5c39b1580 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 39e67c30-8498-4d4c-ae58-f8b7a99c4139 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:08:56.275974Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d3a6c4c5f namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: 8cd6a553-81ac-4235-93c5-357193770d27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d5c4cd1fc namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: de874bcf-aa85-4673-aa99-74b9cd5d26d7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d5de0ce39 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: b2f7156a-3e39-490c-ad96-00264514620b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d60940a77 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: b7161ed4-cf56-418a-aef8-0a7a71fe93d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d60ff89cb namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: 639d8fd3-eb65-491f-91cb-62f8b3103480 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d6108a541 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: e34caec2-9d75-4f18-ba84-bc9eea4c5697 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d639b0a3b namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 48805215-c529-404f-beba-14173d9ee776 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d64070e00 namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: 0dc65bb9-2a25-42c6-b9da-2b75332ef7fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:09:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:33Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62e0d7deac2c4 namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: b75f8a92-8f3e-45d0-8e0f-b967142473c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:26:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:15Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62eff30ca5d90 namespace: kserve-ci-e2e-test resourceVersion: "34372" uid: e7d12119-64bc-4744-9409-6807358465dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 85aed06b-af8a-410d-9c0d-a76d1722e3fd kind: Event lastTimestamp: "2026-09-17T18:26:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:15Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv.18d62eff30cb0438 namespace: kserve-ci-e2e-test resourceVersion: "34373" uid: d9ec206f-2fab-41f1-b1a6-ac5de00c6f33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5ec4c-predictor-68b6c44d66 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 8d56df7a-18f4-4240-b8d1-a51e377bcec8 kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created pod: error-404-isvc-5ec4c-predictor-68b6c44d66-hmwnv' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor-68b6c44d66.18d62e0d393d0872 namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: acee927a-82fb-4044-a5e0-8605b6d28d0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: e6447338-7735-4d86-93a2-2a8b9ba73795 kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Scaled up replica set error-404-isvc-5ec4c-predictor-68b6c44d66 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:56Z" name: error-404-isvc-5ec4c-predictor.18d62e0d36361fe9 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 05655579-21a0-42f9-955a-acf599268d37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 6e5373f3-4123-449a-bf4f-a2d68b392f57 kind: Event lastTimestamp: "2026-09-17T18:09:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:26Z" name: error-404-isvc-5ec4c-predictor.18d62e10b4c76b49 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: 717cf836-dac1-4440-b3ab-e3691254061d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 6e5373f3-4123-449a-bf4f-a2d68b392f57 kind: Event lastTimestamp: "2026-09-17T18:09:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:26Z" name: error-404-isvc-5ec4c-predictor.18d62e10b584ab94 namespace: kserve-ci-e2e-test resourceVersion: "27162" uid: ef0421c5-456e-4303-a212-ae974b221856 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:09:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: 6e5373f3-4123-449a-bf4f-a2d68b392f57 kind: Event lastTimestamp: "2026-09-17T18:10:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:09:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:11Z" name: error-404-isvc-5ec4c-predictor.18d62e17b3427be0 namespace: kserve-ci-e2e-test resourceVersion: "27605" uid: 04485247-06a8-453b-ab35-e6ee5a29cc04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:09:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: 6e5373f3-4123-449a-bf4f-a2d68b392f57 kind: Event lastTimestamp: "2026-09-17T18:10:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:09:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:11Z" name: error-404-isvc-5ec4c-predictor.18d62e17b401615c namespace: kserve-ci-e2e-test resourceVersion: "27606" uid: 6bbaaa04-77f1-41d4-acee-87976cbe2a1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: 512a4a16-257d-4520-9226-ab12f10655e2 kind: Event lastTimestamp: "2026-09-17T18:09:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:43Z" name: error-404-isvc-5ec4c.18d62e0d36415c84 namespace: kserve-ci-e2e-test resourceVersion: "27312" uid: 9e0b1db2-7921-4e75-bdde-c701a97cd9c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27310" uid: 512a4a16-257d-4520-9226-ab12f10655e2 kind: Event lastTimestamp: "2026-09-17T18:09:43Z" message: InferenceService [error-404-isvc-5ec4c] is Ready metadata: creationTimestamp: "2026-09-17T18:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:43Z" name: error-404-isvc-5ec4c.18d62e183a6cdcfb namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: ef7d4ba1-6806-4dee-9494-a979fd8af503 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:48:15.906334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:48:15Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cec6ea197cd namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 149abb0f-88e7-480b-83b6-6a1fdd358fb3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecb43a205e namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 4bad366e-7b90-4c1e-851b-3e97a96cd83e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecb688a3c5 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: fc7491a2-f6c6-40bb-9c50-ef8a52a206d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecb9a3852c namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 91d21cd2-986c-4f96-b9b4-d73593144db4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecb9ffb127 namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: e8d2d940-c266-4b25-81d4-b52c7f727b04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecba089f28 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: f2a80e0a-d361-4d1b-bd4c-b9280d63b209 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecbcd76a25 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 96d7f9de-35cd-4df0-855d-1c5e279b1876 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62cecbd3495cc namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: c9b1a166-34ef-4911-bdfd-cf98098b6834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:33Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62ceccb9178ec namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: de341f69-1b0c-417b-abe8-9c6e2282dcb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62d6d82f51ce1 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 11918fbf-d0bc-45cf-a787-670d2065c7e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62d6d82f6594d namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 8abe3dbc-fc58-4393-83bf-87c4e9d99784 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: e2409f72-3a11-4c01-b722-8b130754e5ae kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:33Z" name: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng.18d62d6e40181895 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 54a0ec7e-31eb-434d-8545-6ff0fafd2332 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8142b-predictor-7948b5d7c6 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 19f75d38-4d28-490c-b026-1b89cec460ae kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: 'Created pod: error-404-isvc-8142b-predictor-7948b5d7c6-zd5ng' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:15Z" name: error-404-isvc-8142b-predictor-7948b5d7c6.18d62cec6d76bad7 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 23af401b-b962-4a5e-91f2-58971169fc51 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: eae11531-2eec-4067-ba98-a4725caeabb9 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Scaled up replica set error-404-isvc-8142b-predictor-7948b5d7c6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:15Z" name: error-404-isvc-8142b-predictor.18d62cec69c2268f namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 940a5626-6ca2-45a2-90a9-80fa2fc80882 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 7ce39820-abbf-4c0f-bf6c-ad9612fd7c54 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: error-404-isvc-8142b-predictor.18d62cefe927b123 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: f13bed98-a47b-45a8-936b-f23ca20f5032 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 7ce39820-abbf-4c0f-bf6c-ad9612fd7c54 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: error-404-isvc-8142b-predictor.18d62cefe9d03816 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 4105e20b-c987-4e1f-8d00-5a512291bcca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 7ce39820-abbf-4c0f-bf6c-ad9612fd7c54 kind: Event lastTimestamp: "2026-09-17T17:49:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:30Z" name: error-404-isvc-8142b-predictor.18d62cf368fd4391 namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: a1b727e4-a9d1-4324-adfe-05834a788ffe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 7ce39820-abbf-4c0f-bf6c-ad9612fd7c54 kind: Event lastTimestamp: "2026-09-17T17:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:30Z" name: error-404-isvc-8142b-predictor.18d62cf369c56ed7 namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 987cb9b1-5d41-4b5c-ab82-95ab18b96e7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8142b namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: 9a17d540-49fd-4c57-b597-471011044df5 kind: Event lastTimestamp: "2026-09-17T17:49:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:03Z" name: error-404-isvc-8142b.18d62cec6a5c7166 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 9c3e34a9-eb28-4c9e-9e0c-7b2a1438fc74 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8142b namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 9a17d540-49fd-4c57-b597-471011044df5 kind: Event lastTimestamp: "2026-09-17T17:49:03Z" message: InferenceService [error-404-isvc-8142b] is Ready metadata: creationTimestamp: "2026-09-17T17:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:03Z" name: error-404-isvc-8142b.18d62cf78b3b7e8b namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 8ed916be-3325-47ae-9f34-e33d920101db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:09:20.303141Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-892d5-predictor-6659f5749d-6qmvb to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:09:20Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e12d28dcda0 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 6f63ce8d-a610-4a4d-87e8-132158fc862e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-892d5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:20Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e12dc16dc8a namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 42da0c43-2f37-4453-8685-8209bf17f09e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27071" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: Add eth0 [10.134.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e13183e77b8 namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: 8bd09046-858c-42dd-b1f4-ae72b41b06fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e1319b4978c namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: c659a038-c5f6-4d4a-96ad-660eaaca51c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e131c927692 namespace: kserve-ci-e2e-test resourceVersion: "27091" uid: f1e8fd9b-61ca-44b5-b74b-abf6ebae0c92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e131d0049dd namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: cd8143f2-5008-4cba-9e04-ed3df78e6341 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e131d097f2b namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: a012591e-6db6-4d31-ab79-9b6877ae35d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e1320d4034d namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: 6a67b2c5-6ece-4e82-a25f-e83644aa85b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e13213fce6f namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 13459ccb-fd84-4834-9b88-66b7987ac743 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:09:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:58Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e1358322d03 namespace: kserve-ci-e2e-test resourceVersion: "27478" uid: 821b8515-b9ed-40a8-8240-8d56cd4e0683 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:18:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:18:34Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e93e63396b2 namespace: kserve-ci-e2e-test resourceVersion: "31170" uid: 68fba9b4-dcfb-45c7-8c6e-9320b8260d3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: a9926232-a078-446e-9461-906e8c3ae850 kind: Event lastTimestamp: "2026-09-17T18:18:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:18:34Z" name: error-404-isvc-892d5-predictor-6659f5749d-6qmvb.18d62e93e63402cc namespace: kserve-ci-e2e-test resourceVersion: "31171" uid: 36bc288d-a7cf-444a-8fd9-fe248845377a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-892d5-predictor-6659f5749d namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: b46f68ad-a3ab-4e6b-99ce-b27cbc6114c7 kind: Event lastTimestamp: "2026-09-17T18:09:20Z" message: 'Created pod: error-404-isvc-892d5-predictor-6659f5749d-6qmvb' metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:20Z" name: error-404-isvc-892d5-predictor-6659f5749d.18d62e12d1e4c5db namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 0ab140e1-afe1-484c-bddf-ef3f77c3615a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 237db40f-744d-434d-a852-258638688407 kind: Event lastTimestamp: "2026-09-17T18:09:20Z" message: Scaled up replica set error-404-isvc-892d5-predictor-6659f5749d from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:20Z" name: error-404-isvc-892d5-predictor.18d62e12cf6a95b5 namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: 7e7409a1-4e3a-4bda-8535-57aa32d68113 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 9eef6fd6-b72c-46dd-a86f-12c0646a9bea kind: Event lastTimestamp: "2026-09-17T18:10:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:05Z" name: error-404-isvc-892d5-predictor.18d62e164de36a3b namespace: kserve-ci-e2e-test resourceVersion: "27535" uid: ec259396-4ea2-46af-8d9c-4060932d86ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 9eef6fd6-b72c-46dd-a86f-12c0646a9bea kind: Event lastTimestamp: "2026-09-17T18:10:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:05Z" name: error-404-isvc-892d5-predictor.18d62e164e3564d4 namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: 8b9ed6be-4b2f-4686-a974-f30dd762f885 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: 9eef6fd6-b72c-46dd-a86f-12c0646a9bea kind: Event lastTimestamp: "2026-09-17T18:10:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:35Z" name: error-404-isvc-892d5-predictor.18d62e20cacc4c6b namespace: kserve-ci-e2e-test resourceVersion: "27817" uid: 26dbe720-be1c-46b2-8ef7-36b853456aea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: 9eef6fd6-b72c-46dd-a86f-12c0646a9bea kind: Event lastTimestamp: "2026-09-17T18:10:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:35Z" name: error-404-isvc-892d5-predictor.18d62e20cb5b9344 namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: ac35cace-40b9-4114-9705-53b5ae628f88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 575ab95e-1bc3-4660-8a56-771595e023d1 kind: Event lastTimestamp: "2026-09-17T18:10:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:10:08Z" name: error-404-isvc-892d5.18d62e12cf6ec048 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 099083a2-523a-4952-98ef-be7598accdbf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 575ab95e-1bc3-4660-8a56-771595e023d1 kind: Event lastTimestamp: "2026-09-17T18:10:08Z" message: InferenceService [error-404-isvc-892d5] is Ready metadata: creationTimestamp: "2026-09-17T18:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:10:08Z" name: error-404-isvc-892d5.18d62e1e156cb84f namespace: kserve-ci-e2e-test resourceVersion: "27569" uid: 3fbc34f3-2726-4438-a67b-d45802c8d2a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:08:09.921313Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9819a-predictor-75b9f568db-gpdmf to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:08:09Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e026f7864d0 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: b4d27ed5-2002-4a7f-9028-714f04bbec6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9819a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:10Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e027b249374 namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 76ba70c4-3c67-47c6-aea2-d53447364301 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02b51c5ab5 namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: d848023a-ec45-4c9a-8071-952738b91db2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02b79db6ce namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: cb8e0c26-a825-4438-869e-9e9ee8247ee8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02ba74ee66 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: cad27231-1bb3-4904-a116-0a2a75f93456 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02bae598e3 namespace: kserve-ci-e2e-test resourceVersion: "26206" uid: 40a73a99-996d-45aa-8590-5f99cadc5605 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02baf003e2 namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 20e8b110-a470-463c-980b-45f39f39e433 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02bf1e7182 namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: c7757768-7d39-4d7c-ba73-d4b75f0a66a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e02bf954204 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: 6c4a5bb1-be77-4dd1-bc37-01dc1b76d544 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:08:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:49Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e03322a6884 namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 634e11c9-7078-43ac-8eec-768d5ba35cab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:09:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:19Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e12bfe68003 namespace: kserve-ci-e2e-test resourceVersion: "27019" uid: 04e12258-45c2-42da-b8fd-679bcc71b3f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 07a60588-4a99-49d3-9ac1-adae9b1f8c69 kind: Event lastTimestamp: "2026-09-17T18:09:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:20Z" name: error-404-isvc-9819a-predictor-75b9f568db-gpdmf.18d62e12bfe6fe45 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: c7f62063-fbc5-4b18-badd-d13e9338fdc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9819a-predictor-75b9f568db namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 2ce69026-0b33-4e4e-82da-5a4f90794afe kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: 'Created pod: error-404-isvc-9819a-predictor-75b9f568db-gpdmf' metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:09Z" name: error-404-isvc-9819a-predictor-75b9f568db.18d62e026e9732ff namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 218c8caa-ef88-46f0-a8fd-0aa461b5aecd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 64f3aa05-b7d1-4eba-98fe-49b08e3b719c kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Scaled up replica set error-404-isvc-9819a-predictor-75b9f568db from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:09Z" name: error-404-isvc-9819a-predictor.18d62e026bef51a3 namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: 9739b05d-fc06-458f-bd85-f70ee78222ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 4ee5b8a9-74c2-4167-aa6e-d7873bc6d59a kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: error-404-isvc-9819a-predictor.18d62e05ea61f48c namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: fb7c31d6-7bb6-4ec5-850c-2d381c7a5445 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 4ee5b8a9-74c2-4167-aa6e-d7873bc6d59a kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: error-404-isvc-9819a-predictor.18d62e05ead8eba6 namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: dff2c18a-81fe-4430-b260-6a4a1a6667cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 4ee5b8a9-74c2-4167-aa6e-d7873bc6d59a kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: error-404-isvc-9819a-predictor.18d62e0ce9440d6d namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: a5e254f2-9c6b-43f5-b383-b81f1abeef4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 4ee5b8a9-74c2-4167-aa6e-d7873bc6d59a kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: error-404-isvc-9819a-predictor.18d62e0cea2c2f6e namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: c3360ccd-e3db-4489-b7f8-a298fe6fe579 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9819a namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: 9ccefef5-7365-4f5b-9354-0884ff320073 kind: Event lastTimestamp: "2026-09-17T18:08:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:59Z" name: error-404-isvc-9819a.18d62e026bd4200c namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: 059342ab-c455-4511-baf5-4d734bfcb504 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9819a namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 9ccefef5-7365-4f5b-9354-0884ff320073 kind: Event lastTimestamp: "2026-09-17T18:08:59Z" message: InferenceService [error-404-isvc-9819a] is Ready metadata: creationTimestamp: "2026-09-17T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:59Z" name: error-404-isvc-9819a.18d62e0df2a1b88b namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: b8ec20ca-fe9d-4e7d-a1a4-25bd3d45040a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:31.109099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c397f-predictor-699fb4686-ws4j5 to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bab92dd24 namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 57faeca3-a6ec-4510-8ae7-79b25a830c74 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bcd670c96 namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 04c4ac98-95bf-465f-9713-e183dd3fb47a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bcf14a993 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 2351b0a0-5594-4628-b2ae-6c43c3efc237 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bd2114d64 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 4e3ea77f-8ad7-4a6d-87f7-c3974f66d687 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bd27fcf81 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 9cc12b16-56de-487c-96a1-d6dbb342c1f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bd288cab7 namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: fb4a5e82-6959-49a6-825b-a1469cb1920c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bd6415f42 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: ecf9b93f-ba93-4574-8b6e-e058200ac346 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bd6ad7797 namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: e3a95eed-10bd-44a7-b31d-fb1718e2e94a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:59:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:58:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:08Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d7bf9653c44 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: d2857083-b520-444a-8160-ec88b3be80eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d8bf42d0a3d namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 3b2d7ebb-61f8-40d4-adc9-2b63a8f2f3ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d8bf42dc3a8 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 4b87463a-9dcb-419e-a607-66aba2351e45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c397f-predictor-699fb4686-ws4j5 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 21b643b5-d440-4a78-8c63-f1870f0a20fd kind: Event lastTimestamp: "2026-09-17T17:59:43Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:43Z" name: error-404-isvc-c397f-predictor-699fb4686-ws4j5.18d62d8c81a57aa2 namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: be339650-e13a-4324-88eb-1ef40eb5cd75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c397f-predictor-699fb4686 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 2bc5a764-d77c-45e4-8846-fca1e6be6f64 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created pod: error-404-isvc-c397f-predictor-699fb4686-ws4j5' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor-699fb4686.18d62d7baa75c645 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 67f02293-0ffc-4a8d-b9b3-4d53d5995231 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 0118b718-114b-4c79-aa57-5cd51ab3a917 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Scaled up replica set error-404-isvc-c397f-predictor-699fb4686 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:31Z" name: error-404-isvc-c397f-predictor.18d62d7ba7a0a94b namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: d424ef03-aa49-419b-b33e-a9d7e37af3c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 80f03070-9f01-4142-9cbe-4a9e5546500d kind: Event lastTimestamp: "2026-09-17T17:58:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:46Z" name: error-404-isvc-c397f-predictor.18d62d7f26603c60 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: d5590e0d-9cb1-4401-9f12-8d2f43cf704b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 80f03070-9f01-4142-9cbe-4a9e5546500d kind: Event lastTimestamp: "2026-09-17T17:58:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:46Z" name: error-404-isvc-c397f-predictor.18d62d7f2725317a namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 3cf317fe-c8a2-4860-a019-f38895bc502e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 80f03070-9f01-4142-9cbe-4a9e5546500d kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:31Z" name: error-404-isvc-c397f-predictor.18d62d82a5e97d91 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 9eb70e2f-1999-4565-ac79-0ccc979603a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 80f03070-9f01-4142-9cbe-4a9e5546500d kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:31Z" name: error-404-isvc-c397f-predictor.18d62d82a6b6c73a namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 130cb954-f8d3-418b-be8b-6f30b6cc3396 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c397f namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: aee498e2-6ba9-43fa-9c55-0347e62a7b4f kind: Event lastTimestamp: "2026-09-17T17:59:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:18Z" name: error-404-isvc-c397f.18d62d7ba7aef2f3 namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 80197a0e-0891-4728-89c6-2018df8b49a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c397f namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: aee498e2-6ba9-43fa-9c55-0347e62a7b4f kind: Event lastTimestamp: "2026-09-17T17:59:18Z" message: InferenceService [error-404-isvc-c397f] is Ready metadata: creationTimestamp: "2026-09-17T17:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:18Z" name: error-404-isvc-c397f.18d62d86b6616bb6 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 98833ae4-841b-4710-b5ed-5a546ae0be9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:16.432017Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:16Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfa863e43b0 namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: acb3d584-2f33-4397-83bd-256645ccb6e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfaa858219e namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 1cbcd997-445c-44c8-9946-72c55427bce5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfaad1e975d namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: c0cc7535-12ef-4af2-b3f6-0ef609066e23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfab01fb49d namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 8d86fd3a-a3d9-4a3d-825c-28dd6a84f9cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfab08fe755 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 7b5ebff2-8595-4815-a92d-da89f2742496 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfab0982cca namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 57ce263b-2fa8-4a87-b690-ca4218729c2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfab35058e7 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 4822238f-fcbc-4e61-a8b6-dfbd484f948c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:49:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:17Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfab3be4dfc namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 0d3fa787-0c3c-4012-9d6d-1e2a3c16374b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:58:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:33Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62cfad565246c namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 5bf295fb-cc04-4132-a843-317023981fd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62d7b9a73bdb6 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 475b0e6b-65b8-4d63-9885-c20bac74e86a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62d7b9a743061 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: e4d30e58-144b-4e85-80d7-72a6f0b73849 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 6c84b656-db79-4375-ab76-87850b55e7b1 kind: Event lastTimestamp: "2026-09-17T17:58:33Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:33Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl.18d62d7c49d7082e namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 69131259-cdda-45f7-9d57-0fd8bef25da4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dcaf3-predictor-5fdbd86d9b namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: 39328237-43a1-4414-8c7d-d60a6cd1ed9f kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: 'Created pod: error-404-isvc-dcaf3-predictor-5fdbd86d9b-xb9bl' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:16Z" name: error-404-isvc-dcaf3-predictor-5fdbd86d9b.18d62cfa8526c7e0 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: 45b2d873-dd9c-48ae-993d-c616ff5efb7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 9ce0db31-2a37-411d-b34f-814b9826f1c9 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Scaled up replica set error-404-isvc-dcaf3-predictor-5fdbd86d9b from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:16Z" name: error-404-isvc-dcaf3-predictor.18d62cfa7fe7b352 namespace: kserve-ci-e2e-test resourceVersion: "16855" uid: fd90da38-5794-4620-89ec-e713d4da714b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: 9edb3c8d-5c26-455e-bed4-4f3c34a7d9d1 kind: Event lastTimestamp: "2026-09-17T17:50:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:01Z" name: error-404-isvc-dcaf3-predictor.18d62cfdfeb22c6c namespace: kserve-ci-e2e-test resourceVersion: "17283" uid: 7859e5ab-a505-4809-963c-3e2760baa184 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: 9edb3c8d-5c26-455e-bed4-4f3c34a7d9d1 kind: Event lastTimestamp: "2026-09-17T17:50:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:01Z" name: error-404-isvc-dcaf3-predictor.18d62cfdff3be8e7 namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: 6064b66c-7407-4e14-8bf5-562828423014 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 9edb3c8d-5c26-455e-bed4-4f3c34a7d9d1 kind: Event lastTimestamp: "2026-09-17T17:50:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:46Z" name: error-404-isvc-dcaf3-predictor.18d62d087d73898e namespace: kserve-ci-e2e-test resourceVersion: "17662" uid: 707d147c-ba6d-4caf-b3fb-bc260081b14f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 9edb3c8d-5c26-455e-bed4-4f3c34a7d9d1 kind: Event lastTimestamp: "2026-09-17T17:50:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:46Z" name: error-404-isvc-dcaf3-predictor.18d62d087e73225e namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: 644d98bb-2596-4aa8-bb1e-e94675d7c4e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: 66db57c4-fff9-4efb-b5d9-711d9c31327d kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:03Z" name: error-404-isvc-dcaf3.18d62cfa80747ede namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 22688b79-2a4a-4e0a-b59e-79ddd0505f2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 66db57c4-fff9-4efb-b5d9-711d9c31327d kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: InferenceService [error-404-isvc-dcaf3] is Ready metadata: creationTimestamp: "2026-09-17T17:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:03Z" name: error-404-isvc-dcaf3.18d62d059563067a namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 19e10ad8-1c0d-4f10-9ab3-af90e8ebb766 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:45.077415Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d to ip-10-0-139-254.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd748ce4ddb namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 32dcb424-ef80-44a1-913a-7ea799fa43ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd76b2586dd namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: b54287c2-4dc2-4d14-82d8-b50a16a17016 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:051af6a7c6b8b08a9503c479397f58819d99aa2694c94c5895c362bcc590737e" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd76d1892aa namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: 8633a5c6-a906-4a78-a3c8-f9e442836c5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd76ff6b898 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 2f1c518a-3ab6-46dd-a1c9-3bd5f47fc684 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd77068b22e namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 2c484dd1-8a1c-4710-ba5e-050feb1a5f21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cd871f19cd7 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 42083099-7cc6-4b3c-b535-47fbbcd52bd5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" in 29.332s (29.332s including waiting). Image size: 1583203731 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdf464e7c90 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: da1506e2-7410-4b2a-a734-d55e730282c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdf4c6d0cfc namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 67b41b96-a40c-40f4-ae33-e70e6598422e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdf4d9bfeab namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: c3b68bf9-0fb7-48a6-979c-3d0600831f73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdf4dab360f namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: c94c7e27-27d9-43f1-b2fe-47a16fa82b26 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.333s (2.333s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdfd8bc813f namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: e55e3917-3aa7-41d5-8ea3-e8873af90e7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdfdd2a16b5 namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: ed3df5aa-7a83-4646-87f3-0130e5ebb0e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cdfde31db10 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 98e9c6e2-c3d0-4662-bc69-786726b0ee6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:47:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:49:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:47:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62ce02f4ec2e9 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: 9a2af45d-a269-4ede-a80c-120c9b4bc808 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cfa660da3a5 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 6195ef02-1e34-44ad-89bf-74c67e8de6f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cfa660e1649 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: 21358576-f54e-47d5-8279-cc94180813e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 86d75845-ca2d-42c4-8c18-426d10aa3a6d kind: Event lastTimestamp: "2026-09-17T17:49:19Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:19Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d.18d62cfb31a57203 namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: efdf5306-2747-45a4-830d-37dc7a4b5940 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6f7b8d4cb namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: b5ad0da2-02de-4a15-9249-17111a4908ca kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6f7b8d4cb-9t97d' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-1-predictor-6f7b8d4cb.18d62cd748359c89 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 945bf3d5-e681-4c46-a818-a61bd7c60e00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: f64f166d-f31c-4c04-b2c9-089468bebf23 kind: Event lastTimestamp: "2026-09-17T17:46:44Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6f7b8d4cb from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:44Z" name: isvc-sklearn-graph-1-predictor.18d62cd741aad31f namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 5d1462d0-6ac9-4f97-9cdf-062ee4299af5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: d6e01c83-fa9a-4298-a053-15e97b627a36 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-sklearn-graph-1-predictor.18d62cdac7cfd3c2 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: cecc5da9-99a3-4d79-bca1-bbaad4b00812 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: d6e01c83-fa9a-4298-a053-15e97b627a36 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-sklearn-graph-1-predictor.18d62cdac85573b6 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 413a7367-86c3-4c77-ab36-f37de8fe3bc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: d6e01c83-fa9a-4298-a053-15e97b627a36 kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-1-predictor.18d62cec47aa9c36 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 8613351f-994d-4fc1-a0a1-09d9d32ed953 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: d6e01c83-fa9a-4298-a053-15e97b627a36 kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-1-predictor.18d62cec487b008e namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: 939ce395-4d92-462f-8317-8c30407d144b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 65b0c5d8-21f2-4138-90b5-17c30ad5682a kind: Event lastTimestamp: "2026-09-17T17:48:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:39Z" name: isvc-sklearn-graph-1.18d62cd74823b664 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 9382a6fc-3163-4168-9b34-0f8713905050 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 65b0c5d8-21f2-4138-90b5-17c30ad5682a kind: Event lastTimestamp: "2026-09-17T17:48:39Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-17T17:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:39Z" name: isvc-sklearn-graph-1.18d62cf1ea4bdb90 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 480544e2-8f1b-403d-889b-1885b294a2da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:45.987921Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-ffc456649-z8txx to ip-10-0-139-254.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd77f13b463 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: f7b734b1-03b3-4f55-8f67-dc4b2e571f1c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd78c4e4d41 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 3ea727e4-1553-4a6a-8110-dcb2b64ddf53 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:47Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd7c56ff4d9 namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: c34f4cb6-30c4-4806-82cc-8eb51aac0575 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:051af6a7c6b8b08a9503c479397f58819d99aa2694c94c5895c362bcc590737e" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:47Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd7c7eb03b8 namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 10b38ac5-bf98-43ba-9bb1-35ac7fe6482a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:47Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd7ca77313d namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: dae9f42a-4c1b-496b-9e11-480afb80673f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:47Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd7caea9f2e namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 03517602-b6aa-4521-96ce-98289ff00d83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cd8ae79b25b namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: ab43e4bd-d03e-4117-9958-f6ba984834f4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" in 27.777s (27.777s including waiting). Image size: 1583203731 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdf261e8fd6 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 384a59d5-a53f-4b45-8d6a-7cad4f46e201 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdf2ba43a63 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: d4f8e268-516b-4181-a68f-79a912362d72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdf2cb5df7a namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: ff54ecb0-aca2-420a-be8f-f7a376e8dae2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdf2cc3a7b3 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: de8f7ced-28b4-4091-8b36-3d13f13f6f54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.884s (2.884s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdfd8b06454 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 012edea8-7e9d-48ba-8aee-251fc22e7800 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdfdd2d82ec namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 1d50e635-324f-44a0-b24c-8a7afe3f38f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdfde31afb6 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: e0bc0bbf-9504-4501-9504-de1fd9dc007c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:49:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cdff34cc4e4 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 868593d0-3a9d-4a6a-bf80-c06ada7a6e41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cfa6171f6d7 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 564e478f-7eb7-43f4-b8af-af9488605e9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cfa6172b07a namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: f757e7de-4c58-40da-8de7-545a3440f807 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 37923647-8dda-442f-8ac8-4f92515839fd kind: Event lastTimestamp: "2026-09-17T17:49:18Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:18Z" name: isvc-sklearn-graph-2-predictor-ffc456649-z8txx.18d62cfaf5cf8378 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 8b2faad7-324d-4e39-8da9-507afe987d73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-ffc456649 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 99c573b4-1574-4c4b-a854-3d3e77d159db kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-ffc456649-z8txx' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-2-predictor-ffc456649.18d62cd77e748008 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 147d88ab-0da1-43bd-b775-e4975b09f5bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 487949de-7723-4274-a208-210298489ecc kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-ffc456649 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: isvc-sklearn-graph-2-predictor.18d62cd774465f39 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 513586d2-6a86-4cb8-bfa2-8488e4f865a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: f694fc7b-7c2c-4343-80b9-dc8a25a0779e kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-sklearn-graph-2-predictor.18d62cdaf2534b91 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: efa019f4-3a7c-4f6a-90ec-d4286287590d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: f694fc7b-7c2c-4343-80b9-dc8a25a0779e kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-sklearn-graph-2-predictor.18d62cdaf3104978 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: e44a7f3b-850d-4fda-8708-126628606f0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: f694fc7b-7c2c-4343-80b9-dc8a25a0779e kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:01Z" name: isvc-sklearn-graph-2-predictor.18d62cec702e491a namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 01488771-70ba-4650-be4c-97d8440bf30a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: f694fc7b-7c2c-4343-80b9-dc8a25a0779e kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:01Z" name: isvc-sklearn-graph-2-predictor.18d62cec7091ad3f namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: d39cde30-3b7f-46a3-ab92-1bd54e2b8a03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 26b8f156-87e7-458f-bd94-e5f148ade251 kind: Event lastTimestamp: "2026-09-17T17:48:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:38Z" name: isvc-sklearn-graph-2.18d62cd773604c21 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 9274d0f8-40f6-44e0-8cb8-95fe93defdf7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 26b8f156-87e7-458f-bd94-e5f148ade251 kind: Event lastTimestamp: "2026-09-17T17:48:38Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-17T17:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:38Z" name: isvc-sklearn-graph-2.18d62cf1ae0cc7a6 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 1d9405f7-ab4c-4c83-8384-b907585361f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:45.620650Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-xkt8s to ip-10-0-139-254.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd7692f65a3 namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 2d8c7f67-0dce-4965-92d5-00af2b54dbcc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd78bc875d9 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: 232682be-2730-41ce-8ff6-e3a36d7fe69c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:051af6a7c6b8b08a9503c479397f58819d99aa2694c94c5895c362bcc590737e" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd78dc8bcb6 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 098949c8-1990-4c7f-ae33-f16358e45a66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd790da332a namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 95f95c17-d93e-4270-8967-0f8a798966af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd7914b8050 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 1585b718-ca69-486d-9d45-f750adbc0c01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cd8aeb9f3ec namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: dfd6e6f4-ffa6-4639-ae7b-b7cf8ac7d0e4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.212s (28.212s including waiting). Image size: 1435742979 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdf4050a462 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 30795744-24df-4db5-a57f-23843c29f546 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdf45f018a5 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 774bb96e-86b3-4226-b37d-5d378a26a6f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdf4721fb45 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 19bc8be4-7075-474a-a394-f062c7dcd3ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdf472f8799 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: d6ad7049-39c5-400c-a77f-6a0b22c2d3d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.454s (2.454s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdfd97ed77a namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: bff4f2c0-dd84-4417-91fb-aa39c96eb96e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdfdd24a423 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 501a8760-d7cf-434c-a6ac-d35364c0595d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:47:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdfde3210a0 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: af21c861-781b-4543-84af-f5a8670cacf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:49:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cdff32d1616 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: dde130d1-becb-43a2-9ece-d4fc95fc70e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cfa6a27ccf4 namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: e1b13921-7a63-40e4-9c29-88084e6863f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cfa6a285897 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 76de4ad4-60e0-4c91-8c10-407cfde95574 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 1e2c904e-062a-49b9-8602-16cd2543955b kind: Event lastTimestamp: "2026-09-17T17:49:18Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-xkt8s.18d62cfaf5cf0a13 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 423fa79b-056c-4170-ad55-be9afbe13da6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-254.ec2.internal source: component: kubelet host: ip-10-0-139-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 657c134f-099f-4862-9ad3-65c7a44479cf kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-xkt8s' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d62cd768893047 namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: d7b19ef3-cd87-407d-b07e-293949d6ea56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: da1fc488-03e6-41f6-9e02-2951a2dee62b kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: isvc-xgboost-graph-predictor.18d62cd7640e1902 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 09b868ff-275e-4eda-963c-871e7cc48356 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 449188a6-61d7-416e-b1e9-3c7d5609f7a5 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-xgboost-graph-predictor.18d62cdae2a5727f namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: 00a75155-8815-4700-88f8-81517b8d0395 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 449188a6-61d7-416e-b1e9-3c7d5609f7a5 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: isvc-xgboost-graph-predictor.18d62cdae3209800 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: fd7bcc13-b1d7-48d1-be1c-00a7b0fcce74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 449188a6-61d7-416e-b1e9-3c7d5609f7a5 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: isvc-xgboost-graph-predictor.18d62cec609bb265 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 21feedab-0455-42ac-ad02-595f0e4e891c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 449188a6-61d7-416e-b1e9-3c7d5609f7a5 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: isvc-xgboost-graph-predictor.18d62cec621e8485 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 8daa17c6-7701-4f97-9746-595c7a549211 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: b98aaf0e-cf4f-47af-80d1-3ff5e7c245a1 kind: Event lastTimestamp: "2026-09-17T17:48:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:28Z" name: isvc-xgboost-graph.18d62cd76431a661 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 416c474a-7f93-40c8-9b7b-124cb5b014aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: b98aaf0e-cf4f-47af-80d1-3ff5e7c245a1 kind: Event lastTimestamp: "2026-09-17T17:48:28Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-17T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:28Z" name: isvc-xgboost-graph.18d62cef59533716 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: da0f004a-48d1-4f82-a0f2-8bc658243ad1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:04.993822Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-597fd9c774-rhswc to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:04Z" name: model-chainer-597fd9c774-rhswc.18d62cf7dc79351b namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 9b9ebca0-ff80-4d3d-b148-778bb758552a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:05Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:05Z" name: model-chainer-597fd9c774-rhswc.18d62cf7fe52cc84 namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 678ca224-e5aa-4ee4-83af-2ff020953491 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:05Z" name: model-chainer-597fd9c774-rhswc.18d62cf800848dac namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 7d5bc6f7-587a-4d8d-ba38-28bf08a0dc0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:05Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-17T17:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:05Z" name: model-chainer-597fd9c774-rhswc.18d62cf8033c28c3 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 32628b35-8dbd-4631-ab67-2613e7bcd4d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:05Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-17T17:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:05Z" name: model-chainer-597fd9c774-rhswc.18d62cf803a99c23 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: a7103d60-f630-4b27-b7d7-779e588d2664 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:15Z" name: model-chainer-597fd9c774-rhswc.18d62cfa5dd8abe5 namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: ead1f210-5d3a-4b67-8546-bbbcc572c738 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-597fd9c774-rhswc namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 566e6261-8653-46d7-bb73-d11160a124cf kind: Event lastTimestamp: "2026-09-17T17:49:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:41Z" name: model-chainer-597fd9c774-rhswc.18d62cfa967b4649 namespace: kserve-ci-e2e-test resourceVersion: "17126" uid: 8940f39e-ba8d-4058-a038-d1e2f7caa4d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-597fd9c774 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 5bcc4648-a663-4ef4-85b3-e9953457f5fa kind: Event lastTimestamp: "2026-09-17T17:49:04Z" message: 'Created pod: model-chainer-597fd9c774-rhswc' metadata: creationTimestamp: "2026-09-17T17:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:04Z" name: model-chainer-597fd9c774.18d62cf7dbaa8653 namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 4309ea3b-c960-4ae8-900e-3fb4ef1bf1f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 90d96214-610f-47c8-a49a-70accb5d51ec kind: Event lastTimestamp: "2026-09-17T17:49:04Z" message: Scaled up replica set model-chainer-597fd9c774 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:04Z" name: model-chainer.18d62cf7da32744f namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: ff538605-e453-454e-a1f1-cd6457e2320f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 5226e676-57fa-413a-9caf-2d0df331eb28 kind: Event lastTimestamp: "2026-09-17T17:49:05Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-17T17:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:05Z" name: model-chainer.18d62cf7e22fe926 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: a0ab5716-47f0-4e36-8b5b-485ac5b3ed33 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:00:41.116264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22661" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1e4e7-6f5ffcc466-vpjqm to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62d99f09ac2c2 namespace: kserve-ci-e2e-test resourceVersion: "22668" uid: 6e55b7e3-fdd2-4f89-be77-e082ae946d8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62d9a12ae441d namespace: kserve-ci-e2e-test resourceVersion: "22684" uid: c35e6f88-ff1c-4fcd-8e1c-37d4905a5456 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1e4e7} kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62d9a142cf9c5 namespace: kserve-ci-e2e-test resourceVersion: "22686" uid: f6d65a56-f609-4b60-b1fb-16d35051101a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1e4e7} kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: 'Created container: sequence-graph-1e4e7' metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62d9a171b3dcb namespace: kserve-ci-e2e-test resourceVersion: "22688" uid: 4d99897b-394d-4ddb-a08d-0130db5b0182 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1e4e7} kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: Started container sequence-graph-1e4e7 metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62d9a178f7076 namespace: kserve-ci-e2e-test resourceVersion: "22689" uid: 0ada3a65-0b6e-4135-852a-fd95e994ed0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1e4e7} kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:08:55Z" message: Stopping container sequence-graph-1e4e7 metadata: creationTimestamp: "2026-09-17T18:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:55Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62e0d1de76c46 namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 87a85907-649c-49f8-9df8-68fc9537e438 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1e4e7} kind: Pod name: sequence-graph-1e4e7-6f5ffcc466-vpjqm namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 71ce48c7-65f2-4590-917b-423b51760da6 kind: Event lastTimestamp: "2026-09-17T18:09:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:23Z" name: sequence-graph-1e4e7-6f5ffcc466-vpjqm.18d62e0da6dd8576 namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: ada3a9d0-11ae-410e-83ee-83b0954b77e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1e4e7-6f5ffcc466 namespace: kserve-ci-e2e-test resourceVersion: "22658" uid: 17b63fb3-9ec9-4cb8-bf73-ecc662c8aee3 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: 'Created pod: sequence-graph-1e4e7-6f5ffcc466-vpjqm' metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7-6f5ffcc466.18d62d99f015b1ac namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 62220d68-a483-43b1-8945-3336ad1175d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22656" uid: 212e516c-53b6-4274-ba6d-9bc42c0528a2 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: Scaled up replica set sequence-graph-1e4e7-6f5ffcc466 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7.18d62d99ef062a42 namespace: kserve-ci-e2e-test resourceVersion: "22659" uid: 65ecace9-2ce1-4f9d-9961-b02dd653b9f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22674" uid: e61127a1-1c70-4aac-bdf5-7fd2256f5889 kind: Event lastTimestamp: "2026-09-17T18:00:41Z" message: InferenceGraph [sequence-graph-1e4e7] is Ready metadata: creationTimestamp: "2026-09-17T18:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:00:41Z" name: sequence-graph-1e4e7.18d62d99f675fa85 namespace: kserve-ci-e2e-test resourceVersion: "22677" uid: 842e0568-a638-4dd6-8282-b4f8f991a77d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: 5ecdff4e-3e7b-465c-a7fc-8667037e3538 kind: Event lastTimestamp: "2026-09-17T18:00:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:56Z" name: sequence-graph-1e4e7.18d62d9d6da952ff namespace: kserve-ci-e2e-test resourceVersion: "22804" uid: 9b4f5e39-000e-46a6-8b8f-3966e2f0693f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: 5ecdff4e-3e7b-465c-a7fc-8667037e3538 kind: Event lastTimestamp: "2026-09-17T18:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:56Z" name: sequence-graph-1e4e7.18d62d9d6e23ddd2 namespace: kserve-ci-e2e-test resourceVersion: "22805" uid: d2a2d49e-9eb2-406a-8a92-791a91fd885c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: 5ecdff4e-3e7b-465c-a7fc-8667037e3538 kind: Event lastTimestamp: "2026-09-17T18:01:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:11Z" name: sequence-graph-1e4e7.18d62da0ecefb25b namespace: kserve-ci-e2e-test resourceVersion: "22913" uid: ba35b541-e4f5-4349-aaf0-7e50e8c9c486 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: 5ecdff4e-3e7b-465c-a7fc-8667037e3538 kind: Event lastTimestamp: "2026-09-17T18:01:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:11Z" name: sequence-graph-1e4e7.18d62da0ed6d5b1e namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: d82ae048-2c17-4a15-825a-c8cb4f1fbc3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:59:30.806512Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c397f-66d6dcb945-6dwwc to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:59:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:59:30Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d8991d10d3b namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 82a44e97-2200-487e-8007-76f7adbeb7b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:59:31Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d89b3d96525 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: ed6851e8-f74b-46df-8d3a-1b5e269da7c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c397f} kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:31Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d89b54ef517 namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: cfc08fc9-7e58-4886-9602-e08b7a3521a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c397f} kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: 'Created container: sequence-graph-c397f' metadata: creationTimestamp: "2026-09-17T17:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:31Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d89b88f97fa namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: 43ac6e48-417c-407b-b4eb-f799672e6fab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c397f} kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T17:59:31Z" message: Started container sequence-graph-c397f metadata: creationTimestamp: "2026-09-17T17:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:31Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d89b8fe12ab namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 8507a7a0-c6e0-43bd-b6f2-9910b30f1ce8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c397f} kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T17:59:40Z" message: Stopping container sequence-graph-c397f metadata: creationTimestamp: "2026-09-17T17:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:40Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d8be9841b20 namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 3b7b6e39-0f4e-40e5-9038-6e4873cafd83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c397f} kind: Pod name: sequence-graph-c397f-66d6dcb945-6dwwc namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 92cd9ea9-793d-48a1-b8fb-cb70ab6edb07 kind: Event lastTimestamp: "2026-09-17T18:00:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:07Z" name: sequence-graph-c397f-66d6dcb945-6dwwc.18d62d8c59727c55 namespace: kserve-ci-e2e-test resourceVersion: "22382" uid: af82bc44-dc3d-4053-ae9a-d648f6e46202 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c397f-66d6dcb945 namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: b7e3b138-834f-4e79-92ed-9d234ad14aae kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: 'Created pod: sequence-graph-c397f-66d6dcb945-6dwwc' metadata: creationTimestamp: "2026-09-17T17:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: sequence-graph-c397f-66d6dcb945.18d62d8990f581d6 namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 09d36167-f1f2-4563-8692-5503e6f5e4c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c397f namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 9fa79ab1-e13e-41ba-89f0-7deaa897659a kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: Scaled up replica set sequence-graph-c397f-66d6dcb945 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: sequence-graph-c397f.18d62d898f5dd49b namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: e582aa4c-0328-4cab-9783-6d496ecda9c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c397f namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 25c11e20-2394-4f6c-a091-ef18d80ed3dc kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: InferenceGraph [sequence-graph-c397f] is Ready metadata: creationTimestamp: "2026-09-17T17:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:30Z" name: sequence-graph-c397f.18d62d8996cf3280 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 737149c0-b1d6-4e00-9a04-eea09a6488ee reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:50:16.005015Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17414" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-dcaf3-8fd969546-vzrkj to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d0865120d0d namespace: kserve-ci-e2e-test resourceVersion: "17418" uid: 80c57ca4-4d3b-4396-a33b-9d3e27773824 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17423" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:50:16Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d0886eaa5fe namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: 1d1baa37-1c3a-407a-9376-387226079eef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dcaf3} kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:50:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d0889ff7996 namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 83515e0d-d4ae-431e-bc58-765497cd68e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dcaf3} kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:50:16Z" message: 'Created container: sequence-graph-dcaf3' metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d088db144c4 namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: 06dcb905-ba2f-4212-ba25-e77c79b1bef2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dcaf3} kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:50:16Z" message: Started container sequence-graph-dcaf3 metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d088e21333a namespace: kserve-ci-e2e-test resourceVersion: "17442" uid: a4c1d55c-cfa3-4936-bdc8-de5231fc0e0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dcaf3} kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Stopping container sequence-graph-dcaf3 metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d7b948bad29 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: acf8b5b7-0923-4763-abf9-0861c4061e2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dcaf3} kind: Pod name: sequence-graph-dcaf3-8fd969546-vzrkj namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 654d0db9-d499-45b7-b36b-8b011aed4640 kind: Event lastTimestamp: "2026-09-17T17:58:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:59Z" name: sequence-graph-dcaf3-8fd969546-vzrkj.18d62d7c5b2e0eed namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 4453b1f8-0624-482b-bdb0-480112e3b500 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-dcaf3-8fd969546 namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: 1280bebb-d3b5-4a9a-aa4a-249f2841cfeb kind: Event lastTimestamp: "2026-09-17T17:50:15Z" message: 'Created pod: sequence-graph-dcaf3-8fd969546-vzrkj' metadata: creationTimestamp: "2026-09-17T17:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:15Z" name: sequence-graph-dcaf3-8fd969546.18d62d086429b809 namespace: kserve-ci-e2e-test resourceVersion: "17416" uid: 8f19126e-8c57-4399-b31c-bbae3a16f474 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: 353089de-a249-4f87-99fd-a638a4acfdf1 kind: Event lastTimestamp: "2026-09-17T17:50:15Z" message: Scaled up replica set sequence-graph-dcaf3-8fd969546 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:15Z" name: sequence-graph-dcaf3.18d62d08617905ec namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: cec328a6-fe21-4c63-af7b-21041facd92f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17424" uid: 1a78071c-122d-4638-a32a-a011065e5228 kind: Event lastTimestamp: "2026-09-17T17:50:16Z" message: InferenceGraph [sequence-graph-dcaf3] is Ready metadata: creationTimestamp: "2026-09-17T17:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:16Z" name: sequence-graph-dcaf3.18d62d0868d061e5 namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 1bee7d49-a3f7-4806-8976-46da3c7093d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: d7f6ea4f-bdbe-4793-83bf-5f0361937479 kind: Event lastTimestamp: "2026-09-17T17:50:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:30Z" name: sequence-graph-dcaf3.18d62d0be1fb31f3 namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 76b32973-8a81-4651-af3d-a7cd9328f7fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: d7f6ea4f-bdbe-4793-83bf-5f0361937479 kind: Event lastTimestamp: "2026-09-17T17:50:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:31Z" name: sequence-graph-dcaf3.18d62d0be2bfe028 namespace: kserve-ci-e2e-test resourceVersion: "17549" uid: 64a57c12-1ef2-4dfa-a3ae-fea533bd7da0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: d7f6ea4f-bdbe-4793-83bf-5f0361937479 kind: Event lastTimestamp: "2026-09-17T17:50:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:46Z" name: sequence-graph-dcaf3.18d62d0f61b036c0 namespace: kserve-ci-e2e-test resourceVersion: "17656" uid: 9d720ff9-3d3a-491f-8e22-bfc08a511fba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: d7f6ea4f-bdbe-4793-83bf-5f0361937479 kind: Event lastTimestamp: "2026-09-17T17:50:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:46Z" name: sequence-graph-dcaf3.18d62d0f6226aa23 namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: 9fe22a52-0346-41f2-a7b9-cd2f0012bc64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:10:20.028789Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-892d5-75f7f9d99c-4gtmq to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e20ba7aea22 namespace: kserve-ci-e2e-test resourceVersion: "27680" uid: eb3b3fa7-1da3-48dd-a0a6-e112240fb3d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27683" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: Add eth0 [10.134.0.56/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e20dc8a2614 namespace: kserve-ci-e2e-test resourceVersion: "27702" uid: ad61b54c-4c69-4529-9a73-3558d6c587fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-892d5} kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e20de02732c namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: 58025e8e-b8ac-474b-afc7-0a2244016de7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-892d5} kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: 'Created container: splitter-graph-892d5' metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e20e157dd89 namespace: kserve-ci-e2e-test resourceVersion: "27707" uid: aa9bdfe3-36c1-4804-aeb8-6689a79c6ce7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-892d5} kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: Started container splitter-graph-892d5 metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e20e1c9b083 namespace: kserve-ci-e2e-test resourceVersion: "27708" uid: 42b3b607-6df4-48b7-a9dd-1ab49e74cf1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-892d5} kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:18:34Z" message: Stopping container splitter-graph-892d5 metadata: creationTimestamp: "2026-09-17T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:18:34Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e93dc646694 namespace: kserve-ci-e2e-test resourceVersion: "31139" uid: 88629311-44dd-46b3-b647-1b51f9f5d241 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:18:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-892d5} kind: Pod name: splitter-graph-892d5-75f7f9d99c-4gtmq namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7d74c01a-17ec-4e13-b0dd-c13940b68d2a kind: Event lastTimestamp: "2026-09-17T18:19:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:19:01Z" name: splitter-graph-892d5-75f7f9d99c-4gtmq.18d62e9466b54017 namespace: kserve-ci-e2e-test resourceVersion: "31373" uid: 528f2e02-0008-4d68-b16d-03ad83de8ca6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-892d5-75f7f9d99c namespace: kserve-ci-e2e-test resourceVersion: "27672" uid: 2269644f-7947-42e7-841f-f30d1da75854 kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: 'Created pod: splitter-graph-892d5-75f7f9d99c-4gtmq' metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5-75f7f9d99c.18d62e20b9e580a3 namespace: kserve-ci-e2e-test resourceVersion: "27676" uid: 5bed526f-f837-4e4b-8f19-3d2b1d393424 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27670" uid: 12265907-544d-455c-970a-1f2096aa47b5 kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: Scaled up replica set splitter-graph-892d5-75f7f9d99c from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5.18d62e20b9294670 namespace: kserve-ci-e2e-test resourceVersion: "27673" uid: 94859f8a-6696-4d50-9578-9b4193ed0323 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27688" uid: 878894b9-7f48-49e5-9095-5227f8b52372 kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: InferenceGraph [splitter-graph-892d5] is Ready metadata: creationTimestamp: "2026-09-17T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:10:20Z" name: splitter-graph-892d5.18d62e20c00d1704 namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: 3b195215-4979-4548-84e0-b7d18bf2a304 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27671" uid: 6eb249eb-c712-4245-8d22-c8c8fec8351e kind: Event lastTimestamp: "2026-09-17T18:10:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:35Z" name: splitter-graph-892d5.18d62e2437aea93f namespace: kserve-ci-e2e-test resourceVersion: "27811" uid: 0ed3394a-dd7e-49f1-8c96-94f40d7fe33c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27671" uid: 6eb249eb-c712-4245-8d22-c8c8fec8351e kind: Event lastTimestamp: "2026-09-17T18:10:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:35Z" name: splitter-graph-892d5.18d62e2438271c86 namespace: kserve-ci-e2e-test resourceVersion: "27813" uid: c0775f4a-423a-4e10-af5a-28618a418b43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:09:09.714011Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9819a-5b6b565754-mn9bp to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:09:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:09:09Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e105b64758c namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 9525f09b-4097-4404-96e0-435c0f995182 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:09Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-9819a-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:09Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e10654f683b namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 890f682c-02ec-4e3d-bafb-326e7c71e531 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:10Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:10Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:09:10Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e10a13ff4b0 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 68823242-82ff-4fd5-b7dd-ed139b4ccb97 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9819a} kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:10Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e10a2f2fbb5 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: c8c8bedd-96c7-4702-85f6-5e5798a97902 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9819a} kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:10Z" message: 'Created container: splitter-graph-9819a' metadata: creationTimestamp: "2026-09-17T18:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:10Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e10a649c1e0 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: fcbde4bd-7991-414c-9473-fd37a6528fe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9819a} kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:10Z" message: Started container splitter-graph-9819a metadata: creationTimestamp: "2026-09-17T18:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:10Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e10a6b73815 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4cf0d000-c30e-42f3-915f-53a2054d35af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9819a} kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:19Z" message: Stopping container splitter-graph-9819a metadata: creationTimestamp: "2026-09-17T18:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:19Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e12b42bbafc namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: a76d5111-43e9-4514-8ea9-66b967df86b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9819a} kind: Pod name: splitter-graph-9819a-5b6b565754-mn9bp namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 91ba2c39-06ab-4248-8a2c-9f4e1aea920a kind: Event lastTimestamp: "2026-09-17T18:09:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:47Z" name: splitter-graph-9819a-5b6b565754-mn9bp.18d62e1354e09f3a namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: 03cddeca-652e-43ac-a188-395b7db70371 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9819a-5b6b565754 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 8b99393a-76e7-498b-9d78-f38a6e6ab8bf kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'Created pod: splitter-graph-9819a-5b6b565754-mn9bp' metadata: creationTimestamp: "2026-09-17T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: splitter-graph-9819a-5b6b565754.18d62e105aca2ddb namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: cd880c8d-1ff0-4529-891a-05e5168f13e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9819a namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 8ef5c2d4-7163-420e-94e5-350fe3bae4ab kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: Scaled up replica set splitter-graph-9819a-5b6b565754 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: splitter-graph-9819a.18d62e1059e1aa63 namespace: kserve-ci-e2e-test resourceVersion: "26852" uid: 4b67bf3b-3a9b-4fda-a6da-0cfc0c81688c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9819a namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: b79d617b-5e98-4edd-917d-dc05ae8b542d kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: InferenceGraph [splitter-graph-9819a] is Ready metadata: creationTimestamp: "2026-09-17T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:09Z" name: splitter-graph-9819a.18d62e1060d96c4f namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 6c425c5b-a839-4a0c-957d-0954082b3cff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:44.903108Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1175d-predictor-c88c69555-c5nkn to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7ee1c2b670 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: d866316b-4ff8-48a9-ad71-b421c0601a95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1175d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:45Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7eedd8a910 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: abccb195-45e0-46da-84fa-c5148f4729ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:46Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:46Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f5d158d09 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 52bf92a4-b25b-434c-a42d-7b7fa2b6bd39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f5ef89d1f namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: c916f63f-33c1-4b4c-8d99-452990eebefc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f627cf7ea namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 172935af-f0ce-4c2f-a87a-5be64b2da4fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f62f7e354 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 408ea372-b7c1-49cb-845d-4ee967856119 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f6303b892 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: eff72f00-9767-45ee-8bd2-8d6c9e2831c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f66674f59 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: f7180ae3-a235-4355-ae67-5f40bb95b1e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:58:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:47Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7f66dbfcd1 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 3e2627d3-20b9-41df-9e13-1d47770720af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T17:59:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:25Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62d7fdb4cf217 namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: 69378a77-c3c3-4d8d-83fd-5749fe274d29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:09Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62e025cda040d namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: d4068c2d-d824-485a-9cee-946f3dafe67a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:09Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62e025cdb4870 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: a29c22c3-af3c-4cc0-94e7-18f2bea5ad97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1175d-predictor-c88c69555-c5nkn namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: b18e7f31-fd64-41dd-b4e4-a8d2542b7d9f kind: Event lastTimestamp: "2026-09-17T18:08:10Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:10Z" name: success-200-isvc-1175d-predictor-c88c69555-c5nkn.18d62e0279b04f9d namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: b9b5a027-5839-48d8-96c7-df77d3770ce2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1175d-predictor-c88c69555 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 332b8825-7076-4b9e-b05a-aa801c89480b kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: 'Created pod: success-200-isvc-1175d-predictor-c88c69555-c5nkn' metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-1175d-predictor-c88c69555.18d62d7ee0c9ed70 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: 344c49f9-27c4-4126-8cef-af7182fdd8af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 0f6d5253-62a0-4f02-b738-28907c8b4ccb kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Scaled up replica set success-200-isvc-1175d-predictor-c88c69555 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-1175d-predictor.18d62d7edcd03155 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: eb518929-0c48-459a-833c-377da4021e94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 9b089605-1629-4584-a5ea-7b94f8f7eabe kind: Event lastTimestamp: "2026-09-17T17:59:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:29Z" name: success-200-isvc-1175d-predictor.18d62d825b6d3ade namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: e487388c-7834-4656-814e-fdf7feb9d2b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 9b089605-1629-4584-a5ea-7b94f8f7eabe kind: Event lastTimestamp: "2026-09-17T17:59:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:29Z" name: success-200-isvc-1175d-predictor.18d62d825c1ff405 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: 354ef2f0-f7eb-4dee-a00d-1e5086b29f84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 9b089605-1629-4584-a5ea-7b94f8f7eabe kind: Event lastTimestamp: "2026-09-17T17:59:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:59Z" name: success-200-isvc-1175d-predictor.18d62d85dc30780b namespace: kserve-ci-e2e-test resourceVersion: "22322" uid: 151fd946-286e-4e2a-aaeb-3cf3829e7a14 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1175d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 9b089605-1629-4584-a5ea-7b94f8f7eabe kind: Event lastTimestamp: "2026-09-17T17:59:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:59Z" name: success-200-isvc-1175d-predictor.18d62d85dd02daa2 namespace: kserve-ci-e2e-test resourceVersion: "22323" uid: 2cbf1684-109f-48bb-873d-c1229409d1b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1175d namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 40d98b84-781a-417b-90f5-9c02cb25d8ff kind: Event lastTimestamp: "2026-09-17T17:59:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:35Z" name: success-200-isvc-1175d.18d62d7edd13c9b3 namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: 8b3f7258-d2de-4c8e-9bd4-9b4849421e3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1175d namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 40d98b84-781a-417b-90f5-9c02cb25d8ff kind: Event lastTimestamp: "2026-09-17T17:59:35Z" message: InferenceService [success-200-isvc-1175d] is Ready metadata: creationTimestamp: "2026-09-17T17:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:35Z" name: success-200-isvc-1175d.18d62d8a97d0c18e namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: 6bdd9725-9853-4032-a24d-6a314beacc0b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:59:41.135457Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1e4e7-predictor-76564f499-76j8q to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8bf97854b5 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 6ad8e641-9fcd-4c9e-9214-4d5e2435abc8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c1b7c2961 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 147b941c-967e-4a2a-902e-335ecb12de77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c1db18819 namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 12d58eea-ac0e-427e-ae64-35867638c865 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c2016d223 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: ca66d3aa-7fff-4943-8b5c-b9b389cfaa7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c208a7637 namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: 9d2a2e64-f105-410d-abb3-6b02f9eb6d95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c20938c2f namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 18b5acad-0328-4aa6-a8e7-f5efc8eea22e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c23209f03 namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: d77bb267-7524-4e64-b9ba-c763ce20d836 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c23931b25 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: 99a06162-ef54-425c-ba10-7f6b642835be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T18:08:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:58Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62d8c3ed51de9 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 10da1965-78f0-43f6-80d6-8e4066272563 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T18:08:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:55Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62e0d28b957d4 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 054c249c-3551-471c-8779-10f82bd32c33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T18:08:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:55Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62e0d28b9e3fe namespace: kserve-ci-e2e-test resourceVersion: "26616" uid: 7fed9c12-afcf-4aaf-be1e-9ed72eb7636b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e4e7-predictor-76564f499-76j8q namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 32d333de-924f-4d2e-a6b7-ac3f05b5fcc1 kind: Event lastTimestamp: "2026-09-17T18:08:58Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:58Z" name: success-200-isvc-1e4e7-predictor-76564f499-76j8q.18d62e0db32d14ed namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: 0e024d60-5575-48b5-a7d6-17dd418d2f51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1e4e7-predictor-76564f499 namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 312b567d-8645-4037-b8e1-3b11197e805f kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: 'Created pod: success-200-isvc-1e4e7-predictor-76564f499-76j8q' metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor-76564f499.18d62d8bf9004445 namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: f494b38e-bd3a-4fad-890f-49be05edb8e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: a415ee89-dcb5-4443-857c-7b5fcf096b0d kind: Event lastTimestamp: "2026-09-17T17:59:41Z" message: Scaled up replica set success-200-isvc-1e4e7-predictor-76564f499 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-1e4e7-predictor.18d62d8bf7207f24 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: a5efcb2e-a338-472d-bdd9-910507b88657 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: cc966645-3282-45f1-b8bf-32a8aac1a209 kind: Event lastTimestamp: "2026-09-17T18:00:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:11Z" name: success-200-isvc-1e4e7-predictor.18d62d8f75f3207f namespace: kserve-ci-e2e-test resourceVersion: "22412" uid: efb66039-b62b-46c4-8dcb-a6d652e7b309 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: cc966645-3282-45f1-b8bf-32a8aac1a209 kind: Event lastTimestamp: "2026-09-17T18:00:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:11Z" name: success-200-isvc-1e4e7-predictor.18d62d8f766d4fa8 namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: de25225d-56e0-458b-a419-a012211413c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: cc966645-3282-45f1-b8bf-32a8aac1a209 kind: Event lastTimestamp: "2026-09-17T18:01:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:11Z" name: success-200-isvc-1e4e7-predictor.18d62d9674df8f75 namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: 6e2c2418-7569-48cf-9790-974242d8a3d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e4e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: cc966645-3282-45f1-b8bf-32a8aac1a209 kind: Event lastTimestamp: "2026-09-17T18:01:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:11Z" name: success-200-isvc-1e4e7-predictor.18d62d96755f3c33 namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: ec290c55-2cdc-4a61-809f-69cf424e0f4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:59:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 5df8db83-f873-493d-a46c-ca6967a76943 kind: Event lastTimestamp: "2026-09-17T18:00:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:00:28Z" name: success-200-isvc-1e4e7.18d62d8bf7296275 namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 4eb39a31-add7-4ac7-ad03-a763f5df247d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e4e7 namespace: kserve-ci-e2e-test resourceVersion: "22554" uid: 5df8db83-f873-493d-a46c-ca6967a76943 kind: Event lastTimestamp: "2026-09-17T18:00:28Z" message: InferenceService [success-200-isvc-1e4e7] is Ready metadata: creationTimestamp: "2026-09-17T18:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:00:28Z" name: success-200-isvc-1e4e7.18d62d96fba957ec namespace: kserve-ci-e2e-test resourceVersion: "22555" uid: 4a4aa489-02ab-421e-812c-121944a543bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:30.306739Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1fda1-predictor-67c979686f-cnskz to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6d83787a01 namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: d57f8e41-65c0-440b-b5c6-07624e12af68 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6da59527cc namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 108ea639-67ab-4c23-a386-b320f5cbbb12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6da733c63e namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 40af3540-2867-43aa-8fe1-d613ce52e26d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6daa158766 namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: a06d0f0c-029d-416b-a164-d799404c70e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6daa8e638e namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 1b06494e-2136-4600-976d-facf4e4506c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6daa99897c namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: aa418cb9-7c3c-4c9e-977f-3fefb94ad111 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6dadad4f0d namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 5b218e35-6917-4ce3-8f33-04ab87ba61c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:57:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:31Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6dae29fae4 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 735573eb-31bf-45a4-b4ff-c47be5f37851 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:58:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:07Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d6dda795b0c namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: bc16555d-585c-45d1-8db6-cd3a7207c724 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d7ed3bd586e namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 3b4ae308-a1c5-4011-b281-c35a32319c71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fda1-predictor-67c979686f-cnskz namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5fbdf638-41bf-4afe-b858-caeb6259a8bf kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-1fda1-predictor-67c979686f-cnskz.18d62d7ed3bdf18e namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 9b0ccf82-0421-48bd-aa85-f94c86eaec34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1fda1-predictor-67c979686f namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 51ca09cd-ce58-47b7-919f-6f0d084df4bb kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created pod: success-200-isvc-1fda1-predictor-67c979686f-cnskz' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor-67c979686f.18d62d6d82eb1b2d namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 08f35425-055a-4078-ab5e-7844be5d5435 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: c1cf6e60-5a80-42aa-a032-58b847ed76be kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Scaled up replica set success-200-isvc-1fda1-predictor-67c979686f from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1-predictor.18d62d6d7b58086b namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 30428363-e391-46cc-bdfd-39205b53f575 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 881e7357-867c-494f-9733-dbbd9a28c6ef kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-1fda1-predictor.18d62d70fae1befe namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 000ba096-55d1-46bd-b1ce-64069d6d7500 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fda1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 881e7357-867c-494f-9733-dbbd9a28c6ef kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-1fda1-predictor.18d62d70fb6e56e3 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: a06a6eb7-5dc7-44ea-b90c-490eb599cedb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 7ee0b7d1-57be-4711-bc10-ac14a95a6279 kind: Event lastTimestamp: "2026-09-17T17:58:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:18Z" name: success-200-isvc-1fda1.18d62d6d7b85837e namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: c77b30b4-1f32-4fd5-888d-16cab75ee16a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 7ee0b7d1-57be-4711-bc10-ac14a95a6279 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-1fda1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1.18d62d6d7c167897 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 0dc04125-dee8-4101-914b-5932bbb7a343 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 7ee0b7d1-57be-4711-bc10-ac14a95a6279 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fda1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-1fda1.18d62d6d7c16c20f namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: cc3c1f8e-4f83-4251-a70b-001f959a6620 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fda1 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 7ee0b7d1-57be-4711-bc10-ac14a95a6279 kind: Event lastTimestamp: "2026-09-17T17:58:17Z" message: InferenceService [success-200-isvc-1fda1] is Ready metadata: creationTimestamp: "2026-09-17T17:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:17Z" name: success-200-isvc-1fda1.18d62d789b1a47d6 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: ce9d684f-6575-444d-a5c7-646ae024cdf2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:45.291531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:45Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cd755918874 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: b8cc0eff-75a7-4be0-8a09-cc0c29961906 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:46Z" message: 'MountVolume.SetUp failed for volume "kube-rbac-proxy-sar-config" : failed to sync configmap cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-17T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:46Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cd79d6e82ef namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: abd857ac-bb32-4fe7-b666-a0a95fc0c58c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:47Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cd7d07fc442 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: a511e9fa-15ab-4b01-abba-6ce7e75187c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:47Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" metadata: creationTimestamp: "2026-09-17T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:47Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cd7d1fb624c namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: af1eb9ee-b010-42a2-83df-617fa0e5d99a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" in 11.382s (11.382s including waiting). Image size: 1356801616 bytes.' metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cda78713cef namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 69c297a7-e575-4a1b-9802-6a5cd6ee7288 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cda7b801b5c namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: b045525b-534b-4772-be97-562bb7d11c7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cda7c0676b9 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: fdec7b6d-560c-4608-a7d2-16e540b82042 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:46:58Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:58Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cda7c142183 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 01e5f6c5-04d1-4e77-b7f0-cb1403aa6134 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:47:01Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.186s (2.186s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:01Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cdafe66e609 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: f6208294-6bf1-40aa-a630-a3c7828ae0ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:47:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:01Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cdb01511991 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 363e4775-edf3-469d-8694-713c5a764a31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:47:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:01Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cdb01cd07ff namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: bbf6a1b8-8e8d-4eac-9108-a2c52499a065 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cdb14cae582 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 8ecb580c-a60f-4d62-8639-6e97b00194f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:15Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cec4dafaadf namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 54c9dc53-f846-43e1-80dc-30aded84778c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:15Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cec4db000f4 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: cc0eff63-ffb9-4363-b3d1-cb6bdc5bd716 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: ed2e2ae3-364f-4f3d-9c1a-0bf5d0003997 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Readiness probe failed: Get "https://10.133.0.16:8643/healthz": dial tcp 10.133.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w.18d62cecc6e9270a namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 1b17cbd1-a8c5-41d7-ab2e-51166721197f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4a129-predictor-5c488f9bbf namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 4d446bc5-0c78-466d-9d27-c8240da4fc0a kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: 'Created pod: success-200-isvc-4a129-predictor-5c488f9bbf-ljp4w' metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: success-200-isvc-4a129-predictor-5c488f9bbf.18d62cd754591c59 namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 237174b8-d09d-40cb-b2f9-8bfd03e0aee1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 13b41487-16ba-4237-950a-2c04f41a8102 kind: Event lastTimestamp: "2026-09-17T17:46:45Z" message: Scaled up replica set success-200-isvc-4a129-predictor-5c488f9bbf from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:45Z" name: success-200-isvc-4a129-predictor.18d62cd750fb6488 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 5a9a30d0-82e6-47f6-9b03-a5cc22458731 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: a121d020-592a-417c-81a4-6babfe291ae2 kind: Event lastTimestamp: "2026-09-17T17:47:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:15Z" name: success-200-isvc-4a129-predictor.18d62cdacfc61933 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: c14cc6c8-f75f-4504-9752-3b76073d2b5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: a121d020-592a-417c-81a4-6babfe291ae2 kind: Event lastTimestamp: "2026-09-17T17:47:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:15Z" name: success-200-isvc-4a129-predictor.18d62cdad0519bc8 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 49f6b0de-84a0-4619-9579-7ec533415ad3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: a121d020-592a-417c-81a4-6babfe291ae2 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: success-200-isvc-4a129-predictor.18d62ce1cef224a3 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: ec83a606-0dad-49b3-9058-d034d62518c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a129-predictor namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: a121d020-592a-417c-81a4-6babfe291ae2 kind: Event lastTimestamp: "2026-09-17T17:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:00Z" name: success-200-isvc-4a129-predictor.18d62ce1cfab3ec4 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: f7862ea5-1f9d-4a01-8341-1c03f749c530 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: df06d955-7134-4fe8-8586-1ab519f18d08 kind: Event lastTimestamp: "2026-09-17T17:47:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:47Z" name: success-200-isvc-4a129.18d62cd750f7230b namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: cc8e34f1-4a28-4f3e-a072-a218fec0125b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: df06d955-7134-4fe8-8586-1ab519f18d08 kind: Event lastTimestamp: "2026-09-17T17:47:47Z" message: InferenceService [success-200-isvc-4a129] is Ready metadata: creationTimestamp: "2026-09-17T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:47Z" name: success-200-isvc-4a129.18d62ce5d3128fd3 namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 5faf440c-30d7-4cd4-a7d0-00aa1c2ac5db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:08:56.047241Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d2cca3115 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 240f4d9e-0f1d-4536-b04c-d82319336656 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d4ec92f86 namespace: kserve-ci-e2e-test resourceVersion: "26689" uid: a4917d83-8484-4bda-9998-3d74487b8444 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d513c5a8f namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: 0b19a38a-8c2a-47f5-83cb-f5fa83061a58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d5425a0ab namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: cbd29e90-2789-4875-8388-bbec2e5c14ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d54a86da3 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 78347da6-8e69-4120-a9a4-d607e6b295e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d54b30710 namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: f454684d-ced1-4e01-86c1-47d3951e6a68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d577b883f namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 595ed1e7-ee2a-4362-a045-ef8e3b6eb074 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0d57eec08c namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 079338d9-877d-4e6d-9a5b-d87b3700d916 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:09:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:34Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62e0db341933f namespace: kserve-ci-e2e-test resourceVersion: "27227" uid: 1ec2c9c3-9e50-4b06-82b5-deaf29c41883 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:26:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:15Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62eff2ded4e2f namespace: kserve-ci-e2e-test resourceVersion: "34356" uid: e0b69ad9-1674-4c4e-b60e-009fea87b81b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cc174225-a213-4d46-8daa-3fb53fb0c9fb kind: Event lastTimestamp: "2026-09-17T18:26:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:15Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt.18d62eff2dedd082 namespace: kserve-ci-e2e-test resourceVersion: "34357" uid: e73abd93-1dcf-4bb7-928c-ae4a34693e94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5ec4c-predictor-5d6f7b484c namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 2c2fa7a2-f404-4821-a839-8c5a299a6abf kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Created pod: success-200-isvc-5ec4c-predictor-5d6f7b484c-9z4xt' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c-predictor-5d6f7b484c.18d62e0d2c3acb6a namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: c30bda02-e0f7-473d-aaf6-2b2b0f4c2a79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 73fbf2e5-e471-4e6f-8b7f-f607a33fb0bc kind: Event lastTimestamp: "2026-09-17T18:08:55Z" message: Scaled up replica set success-200-isvc-5ec4c-predictor-5d6f7b484c from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:55Z" name: success-200-isvc-5ec4c-predictor.18d62e0d29c5bbaa namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: ab6dc0eb-8004-4de4-91d0-774fb8448d64 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 5b474c17-d60b-441c-ad7d-2800d5f50880 kind: Event lastTimestamp: "2026-09-17T18:09:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:26Z" name: success-200-isvc-5ec4c-predictor.18d62e10a8cfce33 namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: ddea2265-3603-4075-b373-714423a9003c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 5b474c17-d60b-441c-ad7d-2800d5f50880 kind: Event lastTimestamp: "2026-09-17T18:09:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:26Z" name: success-200-isvc-5ec4c-predictor.18d62e10a93a7c0c namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: 1869b357-7470-46ca-922a-5c126dab2d47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:09:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 5b474c17-d60b-441c-ad7d-2800d5f50880 kind: Event lastTimestamp: "2026-09-17T18:10:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:09:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:41Z" name: success-200-isvc-5ec4c-predictor.18d62e17a7a0b635 namespace: kserve-ci-e2e-test resourceVersion: "27860" uid: 32dc89de-5f05-4d94-a29b-f9654c10e651 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:09:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5ec4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 5b474c17-d60b-441c-ad7d-2800d5f50880 kind: Event lastTimestamp: "2026-09-17T18:10:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:09:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:41Z" name: success-200-isvc-5ec4c-predictor.18d62e17a8153e41 namespace: kserve-ci-e2e-test resourceVersion: "27861" uid: 52241736-4b61-48d8-9800-5435f370b855 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 6c7b9e61-056d-4fb3-8b91-ce50488b8d5c kind: Event lastTimestamp: "2026-09-17T18:09:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:44Z" name: success-200-isvc-5ec4c.18d62e0d29c405d7 namespace: kserve-ci-e2e-test resourceVersion: "27328" uid: 9bc8c9fb-8d1d-442b-9183-0b3d9137fa2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 6c7b9e61-056d-4fb3-8b91-ce50488b8d5c kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-5ec4c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5ec4c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c.18d62e0d2a655359 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 2a0f6985-06e3-4dd2-92c5-bb3ec919eff0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 6c7b9e61-056d-4fb3-8b91-ce50488b8d5c kind: Event lastTimestamp: "2026-09-17T18:08:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5ec4c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:56Z" name: success-200-isvc-5ec4c.18d62e0d2a659bea namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: c43db8b5-40e8-40e0-8cad-160a9a8a6dd5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: 6c7b9e61-056d-4fb3-8b91-ce50488b8d5c kind: Event lastTimestamp: "2026-09-17T18:09:44Z" message: InferenceService [success-200-isvc-5ec4c] is Ready metadata: creationTimestamp: "2026-09-17T18:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:44Z" name: success-200-isvc-5ec4c.18d62e186fdef696 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: 36cd0dc0-01a3-4e23-8b5c-58823b4767f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:48:15.691643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8142b-predictor-79bffc4977-htq8k to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:48:15Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cec61d5b27c namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 2b79b2c3-38f2-4fec-a1fc-4886cc07b035 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:16Z" message: 'MountVolume.SetUp failed for volume "kube-rbac-proxy-sar-config" : failed to sync configmap cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-17T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:16Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cecac3653af namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: e45f1549-b76b-42d7-b16a-94ba9873d24a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cecdc2f28d9 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 8261ff01-7dd1-4536-abb4-e1f3fb1fe16c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cecde5f34f6 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: f2db0d0a-3995-4167-b8b7-4c11024143a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cece45120d4 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 8657fb8c-af94-4add-8098-5742cd8b5242 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cece4f1997e namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: dcb4aa8c-1562-4561-b0ff-b1f3adb5e058 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cece4fc4e63 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: d03c0193-2630-4257-98d3-1318c43450aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cece8041d9f namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 8f33980c-2077-4c65-9b54-af65e2e5c859 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:17Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62cece86e30ca namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: ca81a7f0-1be5-4b8c-8a6a-7a219c05fdcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:48:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:54Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62ced12c1491c namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 21bb3096-d136-4260-a426-e8c2c29c4b42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62d6d7babcae9 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: e384a908-6ec6-4d1c-902d-fd7b38eb3c5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8142b-predictor-79bffc4977-htq8k namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 4eddbee5-3338-417f-9760-0edcb73635b7 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-8142b-predictor-79bffc4977-htq8k.18d62d6d7bac554f namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 6e53905b-049c-4784-b35b-52d85f749abc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8142b-predictor-79bffc4977 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 6aca9dfb-c3fe-4ca9-8518-1d7fa247321d kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: 'Created pod: success-200-isvc-8142b-predictor-79bffc4977-htq8k' metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:15Z" name: success-200-isvc-8142b-predictor-79bffc4977.18d62cec5f121298 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 6cfcfce6-252f-4c93-86a0-069c7a1e4920 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 671aa824-c2ad-4768-820a-c10e44e16570 kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Scaled up replica set success-200-isvc-8142b-predictor-79bffc4977 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:15Z" name: success-200-isvc-8142b-predictor.18d62cec5af7c963 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 823a3b97-07db-45c8-8492-25119f2b3968 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 2946fbb3-25f9-4638-b664-c8f9f9e1ee13 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: success-200-isvc-8142b-predictor.18d62cefd9bd135c namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 3637c945-4b07-4ac9-afae-2fbc3f467227 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 2946fbb3-25f9-4638-b664-c8f9f9e1ee13 kind: Event lastTimestamp: "2026-09-17T17:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:00Z" name: success-200-isvc-8142b-predictor.18d62cefda4b503b namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 01c0c92e-a348-41e9-afca-b8cbba394706 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 2946fbb3-25f9-4638-b664-c8f9f9e1ee13 kind: Event lastTimestamp: "2026-09-17T17:50:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:00Z" name: success-200-isvc-8142b-predictor.18d62cfa5890fc21 namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: 36556789-4e6b-4579-95c8-961a241fc86f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8142b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 2946fbb3-25f9-4638-b664-c8f9f9e1ee13 kind: Event lastTimestamp: "2026-09-17T17:50:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:00Z" name: success-200-isvc-8142b-predictor.18d62cfa592cfd6d namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: 02ae1271-cb66-4a41-9b0e-aa983b007080 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8142b namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: b68907c5-ef62-4268-be22-9a6c300178aa kind: Event lastTimestamp: "2026-09-17T17:49:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:04Z" name: success-200-isvc-8142b.18d62cec5b5e2213 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 90b9521b-bc35-493b-bb5e-280484a66664 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8142b namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: b68907c5-ef62-4268-be22-9a6c300178aa kind: Event lastTimestamp: "2026-09-17T17:49:04Z" message: InferenceService [success-200-isvc-8142b] is Ready metadata: creationTimestamp: "2026-09-17T17:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:04Z" name: success-200-isvc-8142b.18d62cf7d02c0e8f namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 41a45fe9-042a-4e7e-954d-78384a9a6a6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:09:20.168479Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27039" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-892d5-predictor-b75d9549f-ffcm2 to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:09:20Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e12ca87064e namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 9f7cb51f-9d9b-435d-bd90-008d08f4210e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : failed to sync secret cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-17T18:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:21Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e1312d5abb2 namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: 83744054-818f-4d91-9c6e-cf8bc06faf44 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e1345f65086 namespace: kserve-ci-e2e-test resourceVersion: "27100" uid: 8c056e8d-a7b1-4053-bfce-4b39f097933c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e134744902a namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: c42bd938-ecba-4d8d-a820-379124abc0f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e134a268fc9 namespace: kserve-ci-e2e-test resourceVersion: "27104" uid: a6164f4a-90b4-46e5-86f3-cc989f0d885f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e134a9ab8c8 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: 79c56d55-77b0-4d2d-876c-3c31fd669d27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e134aa5e7b2 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: 6a83dce1-ea78-4526-ac85-f329ec79b8a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e1350d52632 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 7de6c78f-caf0-4003-baea-1ed4b05ea7c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:22Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e13514ed666 namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: 6afc24b2-1a2a-4b8e-950c-e3be2f66f07c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:09:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:09:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:59Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e138b654c69 namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: 1034eb4c-540b-4589-8a08-723559faa1ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:18:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:18:34Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e93e3c65b3b namespace: kserve-ci-e2e-test resourceVersion: "31155" uid: bfe5aadb-7577-474c-84a6-e9d264d607dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 9fd09c19-f729-458e-902d-91771fb05146 kind: Event lastTimestamp: "2026-09-17T18:18:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:18:34Z" name: success-200-isvc-892d5-predictor-b75d9549f-ffcm2.18d62e93e3c7900b namespace: kserve-ci-e2e-test resourceVersion: "31161" uid: ebf92f15-c5b4-45b5-9649-72289dc5e829 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-892d5-predictor-b75d9549f namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 92d56fab-e060-45ed-9ec4-721c75a0368f kind: Event lastTimestamp: "2026-09-17T18:09:20Z" message: 'Created pod: success-200-isvc-892d5-predictor-b75d9549f-ffcm2' metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:20Z" name: success-200-isvc-892d5-predictor-b75d9549f.18d62e12c9b18279 namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: 33f0a46b-4c80-4672-8a50-a3485bf579c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: a1989032-d223-43ee-ad4c-4d80e286f861 kind: Event lastTimestamp: "2026-09-17T18:09:20Z" message: Scaled up replica set success-200-isvc-892d5-predictor-b75d9549f from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:20Z" name: success-200-isvc-892d5-predictor.18d62e12c561f3a8 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 2268c342-b2a8-43fa-b590-0dc67feb1f24 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: d2fa2916-4333-483f-8121-dda332ee5cc2 kind: Event lastTimestamp: "2026-09-17T18:09:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:50Z" name: success-200-isvc-892d5-predictor.18d62e1643ee64ae namespace: kserve-ci-e2e-test resourceVersion: "27370" uid: b6cc154d-e038-4e8c-95cf-ff19f8e6bb8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: d2fa2916-4333-483f-8121-dda332ee5cc2 kind: Event lastTimestamp: "2026-09-17T18:09:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:50Z" name: success-200-isvc-892d5-predictor.18d62e16446b6a32 namespace: kserve-ci-e2e-test resourceVersion: "27371" uid: 864163e4-2c59-4786-9367-b5f200d024d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: d2fa2916-4333-483f-8121-dda332ee5cc2 kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:20Z" name: success-200-isvc-892d5-predictor.18d62e1d42ee9474 namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: 41d5f57f-d2f8-45a9-956d-fb3b478456de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-892d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: d2fa2916-4333-483f-8121-dda332ee5cc2 kind: Event lastTimestamp: "2026-09-17T18:10:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:20Z" name: success-200-isvc-892d5-predictor.18d62e1d437a7e75 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 6e4fff4a-f344-4b54-8bcb-612950e775e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 4983ad40-b0dd-4f1f-a55c-afd154bd46a0 kind: Event lastTimestamp: "2026-09-17T18:10:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:10:09Z" name: success-200-isvc-892d5.18d62e12c5b49db5 namespace: kserve-ci-e2e-test resourceVersion: "27587" uid: 7505e739-fd83-45b8-9804-210188299729 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:10:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-892d5 namespace: kserve-ci-e2e-test resourceVersion: "27585" uid: 4983ad40-b0dd-4f1f-a55c-afd154bd46a0 kind: Event lastTimestamp: "2026-09-17T18:10:09Z" message: InferenceService [success-200-isvc-892d5] is Ready metadata: creationTimestamp: "2026-09-17T18:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:10:09Z" name: success-200-isvc-892d5.18d62e1e493842dc namespace: kserve-ci-e2e-test resourceVersion: "27586" uid: 6e1f8302-f9f0-4468-98fc-899f4cabe63f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:08:09.772771Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:08:09Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02669dc326 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: dd790252-7121-4b6f-a4bc-fc552d94896d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9819a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:10Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e0273cb85cb namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 9dacfcc9-9087-4611-943c-76489362b9de reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02e1a3cd67 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: e1f70367-09e0-4c62-b5db-c2b5fa90737e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02e4483e4d namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 00e7fb05-282b-4c73-95ca-2961e871634b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02e6f91ecd namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: 1808b038-ed9d-456a-a8d2-c7ad035eada5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02e76949d7 namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 81a43057-13bc-4c20-b3b2-1db2930d3a8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02e7722560 namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: a851395c-ce3e-47ff-aa2e-73cedf7237fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02ea375936 namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: 4d672200-fd1f-4699-8f1d-96b5310e92f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:11Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e02eaa773a9 namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 513c7665-bc91-4b12-bc24-d3bfbcedeaec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:08:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:08:49Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e032f80521f namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: fa30ec28-7153-4587-91ef-0b1422bc771a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:09:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:19Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e12bde2e764 namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: 7c902312-ba53-4c56-976e-c580a22a926c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: f98ec658-9c1e-44a5-870d-c315c2612a32 kind: Event lastTimestamp: "2026-09-17T18:09:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:19Z" name: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz.18d62e12bde36458 namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: b97c4df4-d408-4d07-8121-fca554299373 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9819a-predictor-69df5dcdcf namespace: kserve-ci-e2e-test resourceVersion: "26121" uid: f21cff90-5033-4296-8432-764f0b6fbe44 kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: 'Created pod: success-200-isvc-9819a-predictor-69df5dcdcf-bn5vz' metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:09Z" name: success-200-isvc-9819a-predictor-69df5dcdcf.18d62e02662f9c1d namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 5a9af6e5-12a0-4311-93b1-cf8a0bd73f32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: 34c87e2a-1c2a-4952-98de-92f9310ccc77 kind: Event lastTimestamp: "2026-09-17T18:08:09Z" message: Scaled up replica set success-200-isvc-9819a-predictor-69df5dcdcf from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:09Z" name: success-200-isvc-9819a-predictor.18d62e025fd60e5a namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 79ea2810-fa91-4f12-8aca-0a24a4b260e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: e21e8847-fc4a-4801-b074-d778ae1cee53 kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: success-200-isvc-9819a-predictor.18d62e05dfa82205 namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 3b18d976-441d-477c-853e-37c8fa3ef67e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: e21e8847-fc4a-4801-b074-d778ae1cee53 kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: success-200-isvc-9819a-predictor.18d62e05e02ff2fa namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: e404e87c-ff9e-442e-8df5-e1243728a306 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: e21e8847-fc4a-4801-b074-d778ae1cee53 kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: success-200-isvc-9819a-predictor.18d62e0cdf554d31 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 5e5a8b44-d36d-4a2c-8570-2fe13a81d926 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9819a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: e21e8847-fc4a-4801-b074-d778ae1cee53 kind: Event lastTimestamp: "2026-09-17T18:09:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:09Z" name: success-200-isvc-9819a-predictor.18d62e0cdfe19649 namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: a569c1b8-4da4-4f26-a800-d851365ac201 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9819a namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: 0379a7da-42cf-4ec5-86e7-58700a849db0 kind: Event lastTimestamp: "2026-09-17T18:08:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:59Z" name: success-200-isvc-9819a.18d62e025fd6bf56 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: a747ebf7-dc7a-419b-be28-f5c4000c5ef1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9819a namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 0379a7da-42cf-4ec5-86e7-58700a849db0 kind: Event lastTimestamp: "2026-09-17T18:08:59Z" message: InferenceService [success-200-isvc-9819a] is Ready metadata: creationTimestamp: "2026-09-17T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:08:59Z" name: success-200-isvc-9819a.18d62e0dee336062 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: 6443081e-2034-4258-9d53-83580b89b012 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:30.952720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7ba240b611 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: c9feface-cebd-4ecd-bef3-5959b8537243 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bc3eb7429 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 132f1edc-f6c4-4280-8f6e-cc6360b8c4df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bc690436b namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 45eb541c-5595-409f-b40c-eb10aafd08e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bc99bf6d1 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: b97a1c39-6a79-451f-9466-d54680ac5e0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bca1c8428 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 18caaf29-eda2-4afb-a058-d238181ad4d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bca26821d namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: d62e00fb-27c7-44c0-8d2c-3d885034b119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bcd1dd02e namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 2fddf0aa-e9dc-45e0-a675-76b8448833d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:58:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:31Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7bcd8eb5bc namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 0602ac2c-d40a-4e52-9497-2a63a3343205 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:58:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:59:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:58:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:08Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d7be186b1d5 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: e674fe66-8763-4275-a153-57050e5f6133 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:59:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:40Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d8bf107f9c4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: a0819609-d53c-4672-9a0a-86fd3aefe7e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:59:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:41Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d8bf1086224 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: 37698559-1649-4311-83a4-51482257c0d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 48f34ccf-548c-4096-b47c-ff1d35bbb964 kind: Event lastTimestamp: "2026-09-17T17:59:43Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:59:43Z" name: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9.18d62d8c6998a3ad namespace: kserve-ci-e2e-test resourceVersion: "22135" uid: 54d91871-edb5-445b-84d0-f450708e27cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c397f-predictor-84cc68fd6c namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 35b597f9-ccae-49de-8cd0-0b2e5457a2d7 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'Created pod: success-200-isvc-c397f-predictor-84cc68fd6c-lzhj9' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-c397f-predictor-84cc68fd6c.18d62d7ba1bff1a2 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7814d7e5-828c-4dfa-a2c5-208e8652f39c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 61822403-5b12-42ba-a8ef-5b851e7265d9 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Scaled up replica set success-200-isvc-c397f-predictor-84cc68fd6c from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-c397f-predictor.18d62d7b9f2a9559 namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 14598ff3-ce54-4497-b99e-26dc0f2fa25b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 805aaf37-ba1b-4a0b-9127-9ffde4aa3ed9 kind: Event lastTimestamp: "2026-09-17T17:59:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:15Z" name: success-200-isvc-c397f-predictor.18d62d7f1daac14b namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: 96d7db24-3967-4fc3-ba9d-d463c94866ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 805aaf37-ba1b-4a0b-9127-9ffde4aa3ed9 kind: Event lastTimestamp: "2026-09-17T17:59:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:15Z" name: success-200-isvc-c397f-predictor.18d62d7f1e6841b5 namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 5d3add4f-2d61-4a3b-a07a-ac8bd80f9417 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 805aaf37-ba1b-4a0b-9127-9ffde4aa3ed9 kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: success-200-isvc-c397f-predictor.18d62d829d4ea627 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 23e1a711-92f7-4fe2-bcff-b2db6c5348b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:59:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c397f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 805aaf37-ba1b-4a0b-9127-9ffde4aa3ed9 kind: Event lastTimestamp: "2026-09-17T17:59:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:59:30Z" name: success-200-isvc-c397f-predictor.18d62d829dcbdcdc namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 620650fd-ae11-4ef9-a2cd-a7841780a653 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c397f namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: c54dcd98-2f1d-4bc9-a1c1-9017263ed5ff kind: Event lastTimestamp: "2026-09-17T17:59:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:18Z" name: success-200-isvc-c397f.18d62d7b9f214651 namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: 3528bf52-6f36-449c-af5c-9d43716f742e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:59:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c397f namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: c54dcd98-2f1d-4bc9-a1c1-9017263ed5ff kind: Event lastTimestamp: "2026-09-17T17:59:18Z" message: InferenceService [success-200-isvc-c397f] is Ready metadata: creationTimestamp: "2026-09-17T17:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:59:18Z" name: success-200-isvc-c397f.18d62d869fec7afb namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: aa3c4b5b-af23-40da-a980-4a01b3614cf2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:16.175198Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 to ip-10-0-134-158.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa76ef7eb2 namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 5f55e3be-c657-43ed-9cca-683cfc2bd6cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa98a75194 namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: cf06fe80-0721-4325-a882-11bc212f44aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa9af331fe namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: cce8905c-ca5e-43c6-bfc1-684d6bc7d6ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa9d7e3cd8 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 5e44e078-e5ee-46b6-9684-799b10d77910 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa9df6c6eb namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 99a19302-fedb-4fba-a705-54e3b6590cf0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfa9e00906e namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 3f7fd5c1-68ac-4ea8-bd8c-8e076dd21596 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfaa08800fc namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 6e1f3a91-80c4-4744-b188-527b90b946f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfaa0fcbf5a namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: f7cf0f38-86ad-40c3-98b7-c5a105957a1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:49:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:53Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62cfadc76a2f7 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 6aa1a38b-4518-4939-9b0f-1ffc4e8914e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62d7b99aa65e1 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: b14ab0b2-ffb3-4150-8306-84932a1dd0b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 8c7ac24a-084e-4e74-8dcb-6700b25de8c2 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25.18d62d7b99aabee4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: fdb066f3-d42b-4783-84f0-0ee754750818 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-158.ec2.internal source: component: kubelet host: ip-10-0-134-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dcaf3-predictor-5b5b4dfc namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: e239841f-3c8c-4e65-8e52-1d15cf7ce378 kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: 'Created pod: success-200-isvc-dcaf3-predictor-5b5b4dfc-f5n25' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor-5b5b4dfc.18d62cfa75ab77f0 namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: 5390d1dd-2c70-484f-b2a6-ac3d8168549b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: e47dd8dc-4279-4525-883d-8c467a7d3eff kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Scaled up replica set success-200-isvc-dcaf3-predictor-5b5b4dfc from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:16Z" name: success-200-isvc-dcaf3-predictor.18d62cfa7113d277 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: f5766c69-4f75-4d33-88c9-f57c980837e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: cf544822-d5e0-4f4e-8759-b42d70501b7f kind: Event lastTimestamp: "2026-09-17T17:49:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:31Z" name: success-200-isvc-dcaf3-predictor.18d62cfdefa21114 namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 336f81e9-2eef-4258-ad91-7130a4cabf88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: cf544822-d5e0-4f4e-8759-b42d70501b7f kind: Event lastTimestamp: "2026-09-17T17:49:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:31Z" name: success-200-isvc-dcaf3-predictor.18d62cfdf0721031 namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: f9a4d665-c8b4-4f50-a414-604ba5629826 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: cf544822-d5e0-4f4e-8759-b42d70501b7f kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:31Z" name: success-200-isvc-dcaf3-predictor.18d62d016f61af66 namespace: kserve-ci-e2e-test resourceVersion: "17552" uid: 47f4b225-8d43-42d9-87d9-812fd6df8c36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dcaf3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: cf544822-d5e0-4f4e-8759-b42d70501b7f kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:31Z" name: success-200-isvc-dcaf3-predictor.18d62d016fe89d8d namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: d4f9d04d-207d-416e-92f8-81d03b0fefa1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 041b30f2-1d8a-4581-acf7-b731db3a577f kind: Event lastTimestamp: "2026-09-17T17:50:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:04Z" name: success-200-isvc-dcaf3.18d62cfa7155e645 namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: 7505160f-9ee0-4729-b29d-c2254ee087e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dcaf3 namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 041b30f2-1d8a-4581-acf7-b731db3a577f kind: Event lastTimestamp: "2026-09-17T17:50:04Z" message: InferenceService [success-200-isvc-dcaf3] is Ready metadata: creationTimestamp: "2026-09-17T17:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:04Z" name: success-200-isvc-dcaf3.18d62d059f0006c2 namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6affe376-5e5d-4aa7-8441-be7bdf7a314c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:48:05.003467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4a129-8fdcd5874-qjxjz to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:48:05Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62ce9e4c513d4 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 1374b646-d72e-47c3-9618-7f220a821f9d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:05Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:48:05Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cea05ea7103 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 9478e2dd-8009-438d-8d2c-394f8789b95d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:05Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cea076e343d namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 9bbb4c4d-bf72-426f-9d72-20fefe7a14c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" in 1.707s (1.707s including waiting). Image size: 211751512 bytes.' metadata: creationTimestamp: "2026-09-17T17:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:07Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cea6d3202f7 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: 8e36eb9a-eb5e-4ddd-b5f3-fa812cb1cf31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:07Z" message: 'Created container: switch-graph-4a129' metadata: creationTimestamp: "2026-09-17T17:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:07Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cea70155a7b namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: e1ccc4ac-1636-4bb7-9c4b-11eeb74fa0f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:07Z" message: Started container switch-graph-4a129 metadata: creationTimestamp: "2026-09-17T17:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:07Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cea707a1277 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 0688a8c4-4f15-4ab5-92ba-958091dea5eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:15Z" message: Stopping container switch-graph-4a129 metadata: creationTimestamp: "2026-09-17T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:15Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62cec426f58da namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 15c12ad3-0f24-4901-891e-4e87a8c443aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4a129} kind: Pod name: switch-graph-4a129-8fdcd5874-qjxjz namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: b9fbc618-64f3-4112-9eb7-a54419edbb5e kind: Event lastTimestamp: "2026-09-17T17:48:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:48:43Z" name: switch-graph-4a129-8fdcd5874-qjxjz.18d62ced0495e75f namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: f87662ae-c034-423f-ad67-d635fe928092 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4a129-8fdcd5874 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: fb0c841f-557f-42e1-8035-368f771375b4 kind: Event lastTimestamp: "2026-09-17T17:48:04Z" message: 'Created pod: switch-graph-4a129-8fdcd5874-qjxjz' metadata: creationTimestamp: "2026-09-17T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:04Z" name: switch-graph-4a129-8fdcd5874.18d62ce9e372e3d9 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: eeceefa1-f325-42e0-a4e6-4a0e8f4e3d97 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 2da5ec9e-723d-45df-9202-b0f91ad70431 kind: Event lastTimestamp: "2026-09-17T17:48:04Z" message: Scaled up replica set switch-graph-4a129-8fdcd5874 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:04Z" name: switch-graph-4a129.18d62ce9e1d71f8b namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 5068388a-71e9-4877-b7c4-e037c5cca986 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4a129 namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: b47288c0-0ce9-4d6d-b22b-6c6c4ecc0982 kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: InferenceGraph [switch-graph-4a129] is Ready metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:48:05Z" name: switch-graph-4a129.18d62ce9ea58fb3d namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 19c14b1f-be31-4c56-9b6e-e5794e7368ad reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:09:56.172747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27435" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5ec4c-58bffd5c64-54zd4 to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T18:09:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:09:56Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b2c8c82f7 namespace: kserve-ci-e2e-test resourceVersion: "27442" uid: f00a26ab-d9ee-4f7f-9a9d-53e80ff19a12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:56Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:09:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-5ec4c-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:56Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b37a998c4 namespace: kserve-ci-e2e-test resourceVersion: "27456" uid: 7f2ae516-7791-46f3-8923-460366b14362 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27445" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:09:57Z" message: Add eth0 [10.134.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:09:57Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b727a1a92 namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: 2b5b2c0f-8711-4bbb-a01c-216345dc6912 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5ec4c} kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:09:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T18:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:57Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b740e94c4 namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 1c5eaca2-49d1-407f-afc8-b30f798dc377 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5ec4c} kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:09:57Z" message: 'Created container: switch-graph-5ec4c' metadata: creationTimestamp: "2026-09-17T18:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:57Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b7704c7e2 namespace: kserve-ci-e2e-test resourceVersion: "27467" uid: 5180ca1b-f9b3-47c3-a9ae-0009e196ed5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5ec4c} kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:09:57Z" message: Started container switch-graph-5ec4c metadata: creationTimestamp: "2026-09-17T18:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:09:57Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62e1b777390fa namespace: kserve-ci-e2e-test resourceVersion: "27468" uid: 29a02ef3-6b21-4635-a9c3-4d95db87f9a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5ec4c} kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:26:15Z" message: Stopping container switch-graph-5ec4c metadata: creationTimestamp: "2026-09-17T18:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:15Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62eff27123529 namespace: kserve-ci-e2e-test resourceVersion: "34341" uid: 043cae73-76c8-4ea5-84c1-dd31a5c57a14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5ec4c} kind: Pod name: switch-graph-5ec4c-58bffd5c64-54zd4 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0197db6c-fe34-4531-bc88-2854418cc58b kind: Event lastTimestamp: "2026-09-17T18:26:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:26:43Z" name: switch-graph-5ec4c-58bffd5c64-54zd4.18d62efff16e4271 namespace: kserve-ci-e2e-test resourceVersion: "34615" uid: 58a21dde-d9d7-4bbc-82b1-588d719d59b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5ec4c-58bffd5c64 namespace: kserve-ci-e2e-test resourceVersion: "27432" uid: a238ee1b-7711-4f16-8e55-c7ab930ae127 kind: Event lastTimestamp: "2026-09-17T18:09:56Z" message: 'Created pod: switch-graph-5ec4c-58bffd5c64-54zd4' metadata: creationTimestamp: "2026-09-17T18:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:56Z" name: switch-graph-5ec4c-58bffd5c64.18d62e1b2b2e5c79 namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: 622d4473-5938-4bf0-95ff-07188ba24b1f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: f4dbe0f0-b497-45f7-8e38-8c247915f832 kind: Event lastTimestamp: "2026-09-17T18:09:56Z" message: Scaled up replica set switch-graph-5ec4c-58bffd5c64 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:09:56Z" name: switch-graph-5ec4c.18d62e1b299e9642 namespace: kserve-ci-e2e-test resourceVersion: "27433" uid: ef576af3-866d-42b0-86ba-8654f5a95958 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:09:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: 128151b7-980c-4ad9-b6b4-808ee29ffec0 kind: Event lastTimestamp: "2026-09-17T18:09:56Z" message: InferenceGraph [switch-graph-5ec4c] is Ready metadata: creationTimestamp: "2026-09-17T18:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:09:56Z" name: switch-graph-5ec4c.18d62e1b33aaac66 namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: 01efbeb2-b019-4790-a12b-0f1c7c713bb5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27431" uid: 3fcc3c26-06ca-441b-8904-18cde4e44f48 kind: Event lastTimestamp: "2026-09-17T18:10:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:26Z" name: switch-graph-5ec4c.18d62e1ea831b252 namespace: kserve-ci-e2e-test resourceVersion: "27743" uid: 233e1f55-9cef-432c-9c98-82e52b9cd9a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:10:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5ec4c namespace: kserve-ci-e2e-test resourceVersion: "27431" uid: 3fcc3c26-06ca-441b-8904-18cde4e44f48 kind: Event lastTimestamp: "2026-09-17T18:10:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:10:26Z" name: switch-graph-5ec4c.18d62e1ea8cbd06f namespace: kserve-ci-e2e-test resourceVersion: "27744" uid: 9696e415-32ce-4f1d-8283-75097b3c1f49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:15.478145Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8142b-d9d5fcc5c-79qkv to ip-10-0-133-107.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:15Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62cfa4d6362c4 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 54d8d926-06e6-4f28-ab5c-d00ba6b68cdd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fff8674b4-bgl75 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:16Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62cfa6f9a6c10 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: ef004dee-f812-4a6d-9e73-2391ee4bf39c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8142b} kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f6d52988e64cd8a9a21f609c0c944dcdc6613df8ea74ac8dfd85c52003a9bc37" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62cfa73a79a2c namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: cc465383-793c-4919-938b-c58690dd6981 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8142b} kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: 'Created container: switch-graph-8142b' metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62cfa768eb15a namespace: kserve-ci-e2e-test resourceVersion: "16834" uid: 87a82e56-ea42-48c2-ae3d-78a7c44b1ffc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8142b} kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:49:16Z" message: Started container switch-graph-8142b metadata: creationTimestamp: "2026-09-17T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:16Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62cfa76fb1386 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 9ec8dba7-3014-4b6f-a64b-3f0dbd5b2721 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8142b} kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container switch-graph-8142b metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62d6d70011649 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: 0f972f42-75bf-46b1-80e5-97ef0d193a27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8142b} kind: Pod name: switch-graph-8142b-d9d5fcc5c-79qkv namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: fca04026-4b27-41c0-bf37-a4fdde4143ce kind: Event lastTimestamp: "2026-09-17T17:57:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:57Z" name: switch-graph-8142b-d9d5fcc5c-79qkv.18d62d6e15e63651 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 410f4f22-f968-4da7-8936-693ed7e25dde reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-107.ec2.internal source: component: kubelet host: ip-10-0-133-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8142b-d9d5fcc5c namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 85eb2df2-9e8e-4a13-820c-1a3da3dbb016 kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: 'Created pod: switch-graph-8142b-d9d5fcc5c-79qkv' metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:15Z" name: switch-graph-8142b-d9d5fcc5c.18d62cfa4ca6bf75 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 1a93a02b-c67a-422f-8d40-60388e9135bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8142b namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: ea1eae0f-5c5f-4cbb-9c27-61392beeb6e9 kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: Scaled up replica set switch-graph-8142b-d9d5fcc5c from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:15Z" name: switch-graph-8142b.18d62cfa4aefb636 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 36fd75d2-50e1-4c10-9b03-a5af6482e66d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8142b namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 2232a59b-aa94-4a77-b299-f5594ecef3be kind: Event lastTimestamp: "2026-09-17T17:49:15Z" message: InferenceGraph [switch-graph-8142b] is Ready metadata: creationTimestamp: "2026-09-17T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:49:15Z" name: switch-graph-8142b.18d62cfa567b9c4a namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: ca6e97c9-7b9e-4d8c-ac74-519a908e1e25 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8142b namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 0ed622df-535f-4077-abd1-033b4bed5fa0 kind: Event lastTimestamp: "2026-09-17T17:50:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:00Z" name: switch-graph-8142b.18d62cfdc9fa364d namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: 5ad2154d-7e0c-493b-afc9-b87fcefab732 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8142b namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 0ed622df-535f-4077-abd1-033b4bed5fa0 kind: Event lastTimestamp: "2026-09-17T17:50:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:00Z" name: switch-graph-8142b.18d62cfdcaa7b8e0 namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: 7d1ac862-a8ed-4cfe-90cf-6a1e228e8082 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34617"