--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:01:35.802316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-0dcf6-84fd5d8669-w4s5w to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:01:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:01:35Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6ac25b744 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 2fe92989-faec-4262-b1bf-44ce4fca2def reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-0dcf6-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:36Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6b946c09f namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 511acf97-da0f-4bf6-b318-d2cfe4e4f625 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:36Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:01:36Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6f1bf7015 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: f473dfcd-6141-4d52-83ee-bd5dc1d503db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0dcf6} kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:37Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6f3900661 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 095432eb-1e77-43da-940b-9d612e33885f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0dcf6} kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:37Z" message: 'Created container: ensemble-graph-0dcf6' metadata: creationTimestamp: "2026-09-17T18:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:37Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6f5f5c4da namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 00bc84df-c413-4858-aeec-e15337f19d27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0dcf6} kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:37Z" message: Started container ensemble-graph-0dcf6 metadata: creationTimestamp: "2026-09-17T18:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:37Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da6f651f373 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 1b149a72-8754-4bc4-9e85-24447fa56529 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0dcf6} kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:01:45Z" message: Stopping container ensemble-graph-0dcf6 metadata: creationTimestamp: "2026-09-17T18:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:45Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da9079be864 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 298aa6c6-dd12-42ad-9d2e-11d538973bde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0dcf6} kind: Pod name: ensemble-graph-0dcf6-84fd5d8669-w4s5w namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: e1a3d22c-9336-478b-9229-0a4a69c76d9c kind: Event lastTimestamp: "2026-09-17T18:02:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:13Z" name: ensemble-graph-0dcf6-84fd5d8669-w4s5w.18d62da9a224cb32 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: c52a704b-2b37-4be1-8f20-f2ed92d631d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-0dcf6-84fd5d8669 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 7392b3c5-8309-4ddc-8673-53da0252afda kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: 'Created pod: ensemble-graph-0dcf6-84fd5d8669-w4s5w' metadata: creationTimestamp: "2026-09-17T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: ensemble-graph-0dcf6-84fd5d8669.18d62da6abb0a2f8 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 1f998f61-4ef3-4281-9377-965fb7453a2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 2c2b26cb-c6a4-4208-8765-6b48eb07c8fc kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: Scaled up replica set ensemble-graph-0dcf6-84fd5d8669 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: ensemble-graph-0dcf6.18d62da6aaee81de namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: d46f9a79-29fc-4157-9f1e-22c228a1eba2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: a18beb29-63f9-488f-94d0-01dce5351f54 kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: InferenceGraph [ensemble-graph-0dcf6] is Ready metadata: creationTimestamp: "2026-09-17T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:01:35Z" name: ensemble-graph-0dcf6.18d62da6b123634d namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 903b24fa-2376-497f-a7a9-e83b69387feb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:02:46.170132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-9d702-6849cf5859-m6bk5 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:02:46Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db70e65540a namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: 3774a609-3aa8-4811-ba9b-7ba51a1ee994 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:02:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-9d702-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:46Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db71c8f41a5 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: eabc2e8b-a07e-4a90-815d-2b6e03214c60 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:02:47Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:02:47Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db75402fb37 namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: cba05e52-c305-422d-a853-b713d411f975 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9d702} kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:02:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:47Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db7557bdd08 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: 85460da4-ff94-4907-becf-f2d88efea662 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9d702} kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:02:47Z" message: 'Created container: ensemble-graph-9d702' metadata: creationTimestamp: "2026-09-17T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:47Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db758a45d79 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 173a0dfd-3cde-40c7-8587-79537f10d92e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9d702} kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:02:47Z" message: Started container ensemble-graph-9d702 metadata: creationTimestamp: "2026-09-17T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:47Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62db75907036d namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 03fb9244-0208-4139-b948-858366474cf1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9d702} kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Stopping container ensemble-graph-9d702 metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:00Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62e2a37d39d3f namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: ed43f712-dda6-4092-bda0-1ad3bec4578b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-9d702} kind: Pod name: ensemble-graph-9d702-6849cf5859-m6bk5 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e5beaf37-1f94-4f07-8e24-2c92390ec75c kind: Event lastTimestamp: "2026-09-17T18:11:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:28Z" name: ensemble-graph-9d702-6849cf5859-m6bk5.18d62e2aed06c648 namespace: kserve-ci-e2e-test resourceVersion: "25539" uid: ed6f214d-28f4-4f6c-a0e9-563c7cd734fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-9d702-6849cf5859 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 27d94e87-4910-4774-ac1a-12cfffcf8129 kind: Event lastTimestamp: "2026-09-17T18:02:46Z" message: 'Created pod: ensemble-graph-9d702-6849cf5859-m6bk5' metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:46Z" name: ensemble-graph-9d702-6849cf5859.18d62db70debb083 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 6a86ead0-e7c9-44da-99b3-54a33f5a3824 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 71eab5b8-82bf-44f5-bd02-47a08ab7d15d kind: Event lastTimestamp: "2026-09-17T18:02:46Z" message: Scaled up replica set ensemble-graph-9d702-6849cf5859 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:46Z" name: ensemble-graph-9d702.18d62db70d305e4e namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: aabc5261-3853-439d-84a6-5778d41f8fb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 7b8a3003-948b-42c4-83ec-42d6355df273 kind: Event lastTimestamp: "2026-09-17T18:02:46Z" message: InferenceGraph [ensemble-graph-9d702] is Ready metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:46Z" name: ensemble-graph-9d702.18d62db7135cc701 namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: a42a0baa-8493-4259-9120-fe36ee3105c7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 225e6927-2b04-46e5-9624-d02d62968c1e kind: Event lastTimestamp: "2026-09-17T18:03:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:31Z" name: ensemble-graph-9d702.18d62dba8c4933db namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 28ed2725-8cc9-4454-a534-584c709ee4b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 225e6927-2b04-46e5-9624-d02d62968c1e kind: Event lastTimestamp: "2026-09-17T18:03:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:31Z" name: ensemble-graph-9d702.18d62dba8cdca877 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 04a686fd-bc56-494a-8eb1-82c5bc5fc39a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:52:11.342453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-081d8-predictor-7f479c548c-h2qk4 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:52:11Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d233fb7c1cd namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: f664637f-7003-4713-aa6d-423254c303e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-081d8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d234d92334a namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: f46481dd-cf40-4ecc-a157-aa312f4d3a9b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d2385672f41 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 7e2f42b0-5e93-4a72-90c1-a8f1932d370c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d23876b4314 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 17230197-ab45-4009-9143-1f4d01c93662 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d238aa8a5e6 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 821d9be9-25ac-487d-8a3a-bd5987421ad0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d238b199d38 namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: ff263f37-596d-42ad-9428-586f05059c94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d238b23dfb1 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: b5929755-1876-4a73-9178-843832f82061 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d238e320690 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 2c44caf4-c76d-4e5c-8c91-3e4b1c9c1812 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d238ea95609 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: a56aeddf-e1e6-4f70-b05c-7058695a2d4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:52:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T17:52:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:52:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:50Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62d23ecd7469d namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: ed7487a6-4320-4124-ac9a-262de9198496 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T18:01:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:25Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62da461a08e33 namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: fa7633c6-a4d6-4637-8e23-5f13e11065aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b92d1167-af4a-4333-bcfe-73b07e791faf kind: Event lastTimestamp: "2026-09-17T18:01:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:25Z" name: error-404-isvc-081d8-predictor-7f479c548c-h2qk4.18d62da461a1b178 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 35ba51b8-dfdc-43f9-be3a-48d0d12686c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-081d8-predictor-7f479c548c namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: fc2f5f23-7627-440c-b8bf-c3a5cac192ac kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'Created pod: error-404-isvc-081d8-predictor-7f479c548c-h2qk4' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:11Z" name: error-404-isvc-081d8-predictor-7f479c548c.18d62d233f0bf9b0 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: d6c0b63b-86c3-4f71-a3cf-79f6441021a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: b55e9916-0168-4a23-a658-8378e9a30e07 kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Scaled up replica set error-404-isvc-081d8-predictor-7f479c548c from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:11Z" name: error-404-isvc-081d8-predictor.18d62d233d12eefa namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 26bf3258-8526-417a-856d-b11653652c2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: a003927e-ff2c-4513-85c9-a3e7ab18071a kind: Event lastTimestamp: "2026-09-17T17:52:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:41Z" name: error-404-isvc-081d8-predictor.18d62d26bbaee254 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 24eb2b2e-6453-4152-8dd2-f283e4cf7db5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: a003927e-ff2c-4513-85c9-a3e7ab18071a kind: Event lastTimestamp: "2026-09-17T17:52:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:41Z" name: error-404-isvc-081d8-predictor.18d62d26bc20b60e namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 3a8c6e50-86bb-43f2-aacb-486e66046d69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: a003927e-ff2c-4513-85c9-a3e7ab18071a kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: error-404-isvc-081d8-predictor.18d62d2dba7336bf namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 6441f82c-9c3b-4616-914e-5b2f55d39526 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: a003927e-ff2c-4513-85c9-a3e7ab18071a kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: error-404-isvc-081d8-predictor.18d62d2dbadb17f8 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 731e2619-c7ea-46a7-9724-c4e9d3b08c9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 3148a670-ccca-40a4-a9d9-f98e84d5e368 kind: Event lastTimestamp: "2026-09-17T17:53:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:53:00Z" name: error-404-isvc-081d8.18d62d233d2e5ab6 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: fdcee25b-0c6d-4717-880b-af5ca84c8de7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 3148a670-ccca-40a4-a9d9-f98e84d5e368 kind: Event lastTimestamp: "2026-09-17T17:53:00Z" message: InferenceService [error-404-isvc-081d8] is Ready metadata: creationTimestamp: "2026-09-17T17:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:53:00Z" name: error-404-isvc-081d8.18d62d2ea85eb812 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 3226a0e6-9a9e-4c39-9612-76e53a1f1118 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:00:35.956839Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0dcf6-predictor-5d98597747-kxs58 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:00:35Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98bd143e36 namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: 27df5b96-d276-4369-91dd-222000b3e2e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98df030fc1 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 51d3d730-1782-49fe-ac6d-058f94a27f28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e05b7357 namespace: kserve-ci-e2e-test resourceVersion: "19939" uid: 06121d2e-5593-4402-a115-e7100470afb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e2fae2ed namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: 1ba34601-2c01-451a-b88e-4533ece08999 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e354c9ab namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 61fd6e80-d87a-4d97-8e13-a9c5dcce37c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e35e9408 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 13a9e464-2dbe-45e0-96bd-415b7427ec0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e5d2779c namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: 0806c889-4b1a-4070-ac0f-346de1b12161 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d98e6366fcb namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: df9c26ca-b880-47f7-bfe9-0a5374f18c54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:01:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:14Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62d9946bc0dc9 namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 8a86f45e-a408-4095-9d37-20b8fd9acef6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62da90f381150 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 54bbcf54-9fba-40d1-a183-bfc11902ee00 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62da90f39333f namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 3e0d7684-5efc-41c6-9840-06034a03e035 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 43e1e374-0ac1-4fa7-ab65-95ed213ba4ba kind: Event lastTimestamp: "2026-09-17T18:01:50Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-17T18:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:50Z" name: error-404-isvc-0dcf6-predictor-5d98597747-kxs58.18d62daa0a9268ee namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 3f143406-08da-4da0-9d1b-2da2e2a0a5e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0dcf6-predictor-5d98597747 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 13b4c760-1396-48ca-bd6c-fa33852a1469 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: 'Created pod: error-404-isvc-0dcf6-predictor-5d98597747-kxs58' metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:35Z" name: error-404-isvc-0dcf6-predictor-5d98597747.18d62d98bc97135f namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 6ad9801b-0920-4235-a474-493f12f19b16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: 00869b1a-d4c2-4a0e-b66e-006267e04e57 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Scaled up replica set error-404-isvc-0dcf6-predictor-5d98597747 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:35Z" name: error-404-isvc-0dcf6-predictor.18d62d98bacae604 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 2139e524-9bc1-4a1b-94db-ea5978124e18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 6a8b06a6-21e5-4dae-8acd-d8e91db6e222 kind: Event lastTimestamp: "2026-09-17T18:01:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:05Z" name: error-404-isvc-0dcf6-predictor.18d62d9c3925db3b namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: dac1d47f-b653-4fa2-a2cd-ab193d2b6e90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 6a8b06a6-21e5-4dae-8acd-d8e91db6e222 kind: Event lastTimestamp: "2026-09-17T18:01:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:05Z" name: error-404-isvc-0dcf6-predictor.18d62d9c398d0355 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 09dcfd2a-d8e6-41fb-ae97-b897cac7a9f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 6a8b06a6-21e5-4dae-8acd-d8e91db6e222 kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: error-404-isvc-0dcf6-predictor.18d62da33702628e namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: f6262dce-6680-4232-9841-f563224ebee7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 6a8b06a6-21e5-4dae-8acd-d8e91db6e222 kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: error-404-isvc-0dcf6-predictor.18d62da3377446cf namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: e484a9a3-e0f8-412b-ac35-1f1e59f4e981 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: df60a665-43eb-4321-b8b4-56607431c761 kind: Event lastTimestamp: "2026-09-17T18:01:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:01:24Z" name: error-404-isvc-0dcf6.18d62d98bad05fb5 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 223e1d0d-3861-469e-92a1-1c9b355aaccb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: df60a665-43eb-4321-b8b4-56607431c761 kind: Event lastTimestamp: "2026-09-17T18:01:24Z" message: InferenceService [error-404-isvc-0dcf6] is Ready metadata: creationTimestamp: "2026-09-17T18:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:01:24Z" name: error-404-isvc-0dcf6.18d62da402e83544 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: b17ca224-67f9-47b0-9144-aeaeb1570ba8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:51:20.877570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8592d-predictor-94446b57c-nnlbx to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:51:20Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d177fc6ae61 namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 8dd41a21-d066-4cbb-bef8-87371f485b59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a195d8bc namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 7490a7ce-26f7-44c9-af27-b115243e1d37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a31f1421 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: e0a99d76-9517-49e7-899e-ed82077bc130 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a592ed06 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 2786cf52-9192-4e77-8b80-a7019d4dcf75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a5f0db37 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: be12720a-381e-4fab-9bab-ba6545b2c373 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a5fa2174 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 49644bac-81f7-48da-9d83-5e5ea27d7720 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a9008ebd namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: d470dea9-6e82-4f19-8775-2e7a67ab6514 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17a9928f64 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: d3591601-0d81-4d3d-93a6-3f8d5d14208a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T18:00:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:38Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d17c3bb72e2 namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 30e28b96-2d43-40f1-a759-7d11d1254924 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:35Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d98b6f1cf97 namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 7fc2b5b9-271b-491f-9194-5cb476151abf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:35Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d98b6f24009 namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: cc2f270e-aec1-4b90-9d06-c8a9a19991a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8592d-predictor-94446b57c-nnlbx namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: fb8979a8-6a3e-47e6-9c2e-9f59a659a84e kind: Event lastTimestamp: "2026-09-17T18:00:38Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:38Z" name: error-404-isvc-8592d-predictor-94446b57c-nnlbx.18d62d9938262473 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 37b7248a-9009-4518-8766-46e90fcd2cfb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8592d-predictor-94446b57c namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: 4f1be2d1-0386-4aa2-b78b-1c74a4f0d5d2 kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: 'Created pod: error-404-isvc-8592d-predictor-94446b57c-nnlbx' metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:20Z" name: error-404-isvc-8592d-predictor-94446b57c.18d62d177f35bb03 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 8049a51f-ae99-4142-bb8d-64063a6b8b9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: c97f745b-444c-4eb9-b549-8819c7f88a46 kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Scaled up replica set error-404-isvc-8592d-predictor-94446b57c from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:20Z" name: error-404-isvc-8592d-predictor.18d62d177d42943b namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 92d56d90-4bfe-4790-9010-30f41de8fbd1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 34378ad4-90d8-4fda-bdbf-7997e8efac24 kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: error-404-isvc-8592d-predictor.18d62d1afbd841b2 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 4776caf5-513a-4ac1-899c-86609efdc33b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 34378ad4-90d8-4fda-bdbf-7997e8efac24 kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: error-404-isvc-8592d-predictor.18d62d1afc41decc namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: 9b69065b-d2a2-4917-911e-2b7cb85b6e1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 34378ad4-90d8-4fda-bdbf-7997e8efac24 kind: Event lastTimestamp: "2026-09-17T17:52:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:20Z" name: error-404-isvc-8592d-predictor.18d62d21f9dd21cb namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: ba6a438a-8540-4da1-9301-d9a6e67abafa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 34378ad4-90d8-4fda-bdbf-7997e8efac24 kind: Event lastTimestamp: "2026-09-17T17:52:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:20Z" name: error-404-isvc-8592d-predictor.18d62d21fa522f52 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 37a55915-1ca3-41b3-b525-3a68caf4ad93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8592d namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: cd3201ca-ebb4-4780-ae73-b28870616bd9 kind: Event lastTimestamp: "2026-09-17T17:52:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:08Z" name: error-404-isvc-8592d.18d62d177d48bedf namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: f14ccc96-a7fb-4fca-9932-0b0cff2d955e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8592d namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: cd3201ca-ebb4-4780-ae73-b28870616bd9 kind: Event lastTimestamp: "2026-09-17T17:52:08Z" message: InferenceService [error-404-isvc-8592d] is Ready metadata: creationTimestamp: "2026-09-17T17:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:08Z" name: error-404-isvc-8592d.18d62d2286c53d29 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: ab05baae-e4d1-454b-9f31-cc76239fc896 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:11:51.290521Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-85bfa-predictor-5d4d479565-4dlks to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e35fa1a73a3 namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 7f3dc4f2-c7db-4cb6-8704-ea028d207fe2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e361c1e73ae namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 2929fef0-48b5-4b7f-9413-d285ae11eed9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e361d84cd82 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: d919f564-d4a9-4104-8666-a315f1eceda9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e362073b25a namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: fe563de2-e306-4522-9a46-74707ef5d6ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e3620e75e64 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 20c59c89-6d6f-4c85-8612-9453af566baf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e3620f1dba9 namespace: kserve-ci-e2e-test resourceVersion: "26005" uid: 90df7158-c44d-4f2f-8434-a91540eec2d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e3623b95723 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: ce148943-9745-4fe5-bbe0-0053b3c3560f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e362429fad5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: aa90bdb5-2b07-432a-9856-7fba4707b6bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:12:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:28Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62e36271c011b namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 9fb7a924-2a9d-40f5-91fe-4b95b41f326a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:29:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:10Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62f27e99c6587 namespace: kserve-ci-e2e-test resourceVersion: "33643" uid: 519ad6a7-ef30-49f5-8eca-65f6d30d7817 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:29:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:10Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62f27e99cf9ac namespace: kserve-ci-e2e-test resourceVersion: "33645" uid: 425a5e38-1e4a-4654-afb8-1ede95c25bb8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:29:13Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:13Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62f2887e8686d namespace: kserve-ci-e2e-test resourceVersion: "33695" uid: daa86bed-4356-4be2-84f6-03d368cde875 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:29:19Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-09-17T18:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:19Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62f29ed8967fe namespace: kserve-ci-e2e-test resourceVersion: "33746" uid: 0208bb64-bc3e-44c1-bdf3-7a180507f5af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 1219c9c7-e3bb-408a-99aa-7cc5583a5cdf kind: Event lastTimestamp: "2026-09-17T18:29:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: i/o timeout' metadata: creationTimestamp: "2026-09-17T18:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:19Z" name: error-404-isvc-85bfa-predictor-5d4d479565-4dlks.18d62f29edda6ff8 namespace: kserve-ci-e2e-test resourceVersion: "33747" uid: ff3b72e4-d0f5-48c2-a3e1-1b4e3b1a21ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-85bfa-predictor-5d4d479565 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 813ab82c-d2e8-4bb4-a03c-41ffae3a761a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created pod: error-404-isvc-85bfa-predictor-5d4d479565-4dlks' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor-5d4d479565.18d62e35f98d1d99 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 5802d612-e2cc-4ded-a61e-45674e172bc9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: 01630e32-1153-4ea7-bb7a-40076e154492 kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Scaled up replica set error-404-isvc-85bfa-predictor-5d4d479565 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-85bfa-predictor.18d62e35f8089a5a namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 6368040b-5fae-4620-a32e-1dd96ac11219 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 1a9b6e73-bf78-4665-b4a0-dcbab3f3b1c2 kind: Event lastTimestamp: "2026-09-17T18:12:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:06Z" name: error-404-isvc-85bfa-predictor.18d62e397677fd9c namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: 680a1e31-9272-4fc5-8624-c1961dc417a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 1a9b6e73-bf78-4665-b4a0-dcbab3f3b1c2 kind: Event lastTimestamp: "2026-09-17T18:12:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:06Z" name: error-404-isvc-85bfa-predictor.18d62e3976de3c3a namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 184f25a8-ad21-488f-a6f6-7d516d1f7f11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: 1a9b6e73-bf78-4665-b4a0-dcbab3f3b1c2 kind: Event lastTimestamp: "2026-09-17T18:13:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:21Z" name: error-404-isvc-85bfa-predictor.18d62e3cf5a26096 namespace: kserve-ci-e2e-test resourceVersion: "27049" uid: fe90b273-dbd1-45c1-99a8-faf7ae02da5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: 1a9b6e73-bf78-4665-b4a0-dcbab3f3b1c2 kind: Event lastTimestamp: "2026-09-17T18:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:21Z" name: error-404-isvc-85bfa-predictor.18d62e3cf619f5f2 namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: e6ea0276-533d-4c7d-a0aa-49fd06f1f0c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-85bfa namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 86decde5-3658-41eb-9c79-8c50fac6e2dc kind: Event lastTimestamp: "2026-09-17T18:12:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:38Z" name: error-404-isvc-85bfa.18d62e35f80f0596 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 8e15f808-cad6-48aa-845e-72050022d46f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 86decde5-3658-41eb-9c79-8c50fac6e2dc kind: Event lastTimestamp: "2026-09-17T18:12:38Z" message: InferenceService [error-404-isvc-85bfa] is Ready metadata: creationTimestamp: "2026-09-17T18:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:38Z" name: error-404-isvc-85bfa.18d62e40e2a84b25 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 14df43b9-677a-4b55-87f1-1473ce103306 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:01:26.161433Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8649f-predictor-f69888d4c-dvxpd to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da46d818999 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 0a6f4fb0-8e7f-436b-83e0-17310d773b1a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da48f2ab727 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: dc2d0026-b346-49bf-a1a1-21fd6c6faae5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da490947d13 namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 59857990-ab18-460a-b262-f08d56da4952 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da493166363 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: b00dacea-4265-4a56-b1c7-900f27c76af6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da49375d43f namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 65244d86-583a-4ea2-8e23-845f75da0702 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da4937f7e37 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: a9e1c727-9b26-4584-8bc7-dd0177ff3079 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da495e9a5c4 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 220fff76-ae5f-4d75-b221-b426e9938b10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da49647b1ce namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c23dfdc5-5f9e-4feb-bfef-db78bfde445e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:02:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:03Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62da4bb5f2717 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 0c24e05f-824b-418a-b0b9-1288a38532b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62db4bdcaa9d9 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: 84de254d-5dce-44bc-aafb-923b5e737c57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62db4bdcb1e4b namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: d8d8b8a6-2381-4a7c-90b6-bd7580f91fa3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 74c1ac49-8a02-45ec-b881-59bbb009d858 kind: Event lastTimestamp: "2026-09-17T18:02:38Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:38Z" name: error-404-isvc-8649f-predictor-f69888d4c-dvxpd.18d62db5437daf56 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 400a1114-086c-4729-b26a-093c632fa8c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8649f-predictor-f69888d4c namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 88b74a70-5284-4ad2-88b2-bfe1c0f6d4e8 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: 'Created pod: error-404-isvc-8649f-predictor-f69888d4c-dvxpd' metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor-f69888d4c.18d62da46d0de5a2 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 9cd0279a-bcf0-4ea0-9cf3-e2287ee502b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 0fb3f421-c015-434f-86bf-1209d273b17d kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Scaled up replica set error-404-isvc-8649f-predictor-f69888d4c from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:26Z" name: error-404-isvc-8649f-predictor.18d62da46b72934d namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 6e9b860e-3e8b-4c5c-aef6-263b38e6813a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: dcd1c3ba-6fda-4dae-91e9-73d402c6a917 kind: Event lastTimestamp: "2026-09-17T18:01:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:56Z" name: error-404-isvc-8649f-predictor.18d62da7e9bd2859 namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 8f3bf817-f579-4832-bd59-2500518774e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: dcd1c3ba-6fda-4dae-91e9-73d402c6a917 kind: Event lastTimestamp: "2026-09-17T18:01:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:56Z" name: error-404-isvc-8649f-predictor.18d62da7ea23b0fe namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 50aedb32-f66f-437e-abf0-fa6461465363 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: dcd1c3ba-6fda-4dae-91e9-73d402c6a917 kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: error-404-isvc-8649f-predictor.18d62daee7fc17a2 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: 5836348d-e7a9-460d-ac39-1ef70184a88c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: dcd1c3ba-6fda-4dae-91e9-73d402c6a917 kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: error-404-isvc-8649f-predictor.18d62daee8b3287b namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: b4b7edaf-2094-430c-8f5c-1678dece5311 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8649f namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 44b55689-4ab1-49e1-a40a-0d00a886b731 kind: Event lastTimestamp: "2026-09-17T18:02:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:13Z" name: error-404-isvc-8649f.18d62da46b3eee0e namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 1fc6ac43-bfb8-4aee-a4d2-5663b11419ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8649f namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 44b55689-4ab1-49e1-a40a-0d00a886b731 kind: Event lastTimestamp: "2026-09-17T18:02:13Z" message: InferenceService [error-404-isvc-8649f] is Ready metadata: creationTimestamp: "2026-09-17T18:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:13Z" name: error-404-isvc-8649f.18d62daf76f647d7 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 2dde1662-73f2-45ad-b8d5-5bb1f29aa41c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:02:36.414559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8eede-predictor-5d489c9599-98jkr to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db4c8eb2aa4 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 1bb0f945-fb04-4c41-8793-bd8d2583201a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8eede-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db4d580709a namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 0f702f7b-4399-400f-938e-d6c5eb292c29 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db50e916a77 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 30e2bab2-66c3-4617-8dc4-b1e0b3e4b9c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db510d92073 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 66f545fb-a0e6-4582-8911-9ec6c8c97a2d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db51425fc5f namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 4d69bdc3-4f8d-4d49-9d87-eb2c30f28ce9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db514873b18 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 10cc4e50-36bc-48d3-a0c6-fe0aab7cc340 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db51490d1c4 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: e1df3a6b-9bd1-430d-8e12-c5d32957f8a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db516e8e355 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: a5bb1e6b-10a0-49a4-8035-4891d06ab4d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db51744bafc namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: d9ea6326-3be0-42c6-9b71-d7ceabe39e66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:03:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:14Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62db555dec0c8 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: 182cdca2-3321-455e-98f6-f4e6a510357d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62e35ee12af43 namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: f51abde1-c859-4e3a-8154-29e4c04be033 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8eede-predictor-5d489c9599-98jkr namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 73a12907-2862-4ed2-9a2b-ef500261948f kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: error-404-isvc-8eede-predictor-5d489c9599-98jkr.18d62e35ee133a49 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 727c8d7b-719e-406c-a1dd-6eba75cada82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8eede-predictor-5d489c9599 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 1229e6f0-0d62-44e4-87dc-e26e3f20c5cb kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: 'Created pod: error-404-isvc-8eede-predictor-5d489c9599-98jkr' metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8eede-predictor-5d489c9599.18d62db4c87d2c79 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 5b248ea6-94ef-4f86-8070-a848cf7a2f59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 25d27763-03d9-43b7-84bb-454ac3fb45e8 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Scaled up replica set error-404-isvc-8eede-predictor-5d489c9599 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:36Z" name: error-404-isvc-8eede-predictor.18d62db4c6b7e088 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 352eb468-5896-4d66-bb2e-b773bb3c4fcf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 76bf1430-b9c4-4a6f-82c9-aa04e7dc6c5f kind: Event lastTimestamp: "2026-09-17T18:03:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:06Z" name: error-404-isvc-8eede-predictor.18d62db84545f029 namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: d7440847-06cd-4339-ba47-ae9f1b29a8f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 76bf1430-b9c4-4a6f-82c9-aa04e7dc6c5f kind: Event lastTimestamp: "2026-09-17T18:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:06Z" name: error-404-isvc-8eede-predictor.18d62db845a35f74 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: 7b14a84d-08f1-4b4a-93c1-e9c15b0e9da7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 76bf1430-b9c4-4a6f-82c9-aa04e7dc6c5f kind: Event lastTimestamp: "2026-09-17T18:03:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:51Z" name: error-404-isvc-8eede-predictor.18d62dbf436d0f10 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: f797fadd-42c5-4206-b6c8-89c96c2e3437 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 76bf1430-b9c4-4a6f-82c9-aa04e7dc6c5f kind: Event lastTimestamp: "2026-09-17T18:03:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:51Z" name: error-404-isvc-8eede-predictor.18d62dbf43ec3472 namespace: kserve-ci-e2e-test resourceVersion: "22148" uid: 144741c4-6b82-4061-bc47-4ca320430090 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8eede namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 75fd0381-6e91-4d77-9586-1e090ac24523 kind: Event lastTimestamp: "2026-09-17T18:03:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:03:24Z" name: error-404-isvc-8eede.18d62db4c6a5859c namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 0e8cdb57-7373-4d27-9c0f-5970686f5895 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8eede namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 75fd0381-6e91-4d77-9586-1e090ac24523 kind: Event lastTimestamp: "2026-09-17T18:03:24Z" message: InferenceService [error-404-isvc-8eede] is Ready metadata: creationTimestamp: "2026-09-17T18:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:03:24Z" name: error-404-isvc-8eede.18d62dc012f441bd namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 21b88264-9d47-413c-938f-57246c12d2bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:01:46.254912Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9d702-predictor-5967c4d597-nkqp8 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da91b2ba973 namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 88e9855c-003e-41d1-92bf-8062c5514740 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9d702-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da9248d41c6 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: b387bfee-8913-4247-b3bf-d68190dfa0c5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da960b51bf5 namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: f6fee524-a3ad-42e4-8fcf-9a3ddc95a7b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da9622b2d45 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: c3386d36-48c4-410d-9d9a-5ffe2504ad01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da96515b842 namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 32f3b1f0-f254-4087-8724-83ea10518685 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da96596cc61 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 10353297-d106-4f6e-b939-d8b94163fce6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da965a00bea namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: f44984e9-9fc5-4762-8f42-2aa540aa1395 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da9683047fa namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 4ecc4bc2-9c51-47b8-8952-6525497a4130 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da968daf366 namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: ff9539b6-3922-43db-baa5-d8462a71754e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:02:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:25Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62da9e0cd4e3d namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: c9b24afc-767e-4438-a6fb-0975ebc6a145 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:00Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62e2a42e67636 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: 4149d0a5-ff7e-402c-9254-ad00b9ede95d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: f161c155-a751-4207-a686-6154fb8949a9 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:00Z" name: error-404-isvc-9d702-predictor-5967c4d597-nkqp8.18d62e2a42e7187d namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: 024efe18-2dff-46f3-97c7-d5b8db934d58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9d702-predictor-5967c4d597 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: a0fb6658-ea39-431e-93be-0c32e5ea9236 kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: 'Created pod: error-404-isvc-9d702-predictor-5967c4d597-nkqp8' metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-9d702-predictor-5967c4d597.18d62da91a95c423 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 76653bbd-5d41-4207-9b81-39d3cdeac218 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: d8521556-2997-4163-a8ad-49270d915abd kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Scaled up replica set error-404-isvc-9d702-predictor-5967c4d597 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:46Z" name: error-404-isvc-9d702-predictor.18d62da918dc6371 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: f003bdd9-895c-4b3c-bc56-278018553495 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:02:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: 38a6303e-c7a5-419e-89de-f4e5d6d50122 kind: Event lastTimestamp: "2026-09-17T18:02:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:31Z" name: error-404-isvc-9d702-predictor.18d62dac975219d7 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 557bf498-c286-494f-821f-a21d5470a99e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:02:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: 38a6303e-c7a5-419e-89de-f4e5d6d50122 kind: Event lastTimestamp: "2026-09-17T18:02:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:31Z" name: error-404-isvc-9d702-predictor.18d62dac97b90e60 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: 2cf086cc-4705-46be-b646-5991d88b07a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 38a6303e-c7a5-419e-89de-f4e5d6d50122 kind: Event lastTimestamp: "2026-09-17T18:03:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:16Z" name: error-404-isvc-9d702-predictor.18d62db713f25467 namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: c859595d-1d19-4edd-a87f-1ed7c53c5dba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:02:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 38a6303e-c7a5-419e-89de-f4e5d6d50122 kind: Event lastTimestamp: "2026-09-17T18:03:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:16Z" name: error-404-isvc-9d702-predictor.18d62db7143b7958 namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: 49afa888-f765-4369-99e0-2f652ad69a7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d702 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: ed56f5e1-6f2c-40af-9169-dbcef1e39b32 kind: Event lastTimestamp: "2026-09-17T18:02:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:35Z" name: error-404-isvc-9d702.18d62da918b16d9f namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 43b669c5-80de-4b98-9cc9-cd0c3846d085 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: ed56f5e1-6f2c-40af-9169-dbcef1e39b32 kind: Event lastTimestamp: "2026-09-17T18:02:35Z" message: InferenceService [error-404-isvc-9d702] is Ready metadata: creationTimestamp: "2026-09-17T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:35Z" name: error-404-isvc-9d702.18d62db49c812e1e namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: d6ed73ca-b84d-402f-a5ed-96d106e5390a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:50.752866Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:50Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d0283ecd487 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: cf2510fa-7522-4b16-ad0a-e65ef2057cfd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d46ec-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:50Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d028e1d0902 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 8aee0b4b-5997-4451-8f7c-a40c6152bcec reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:51Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d02c9dd1201 namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: ea4d7f6f-8406-445f-83a9-adbb52ec57ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d02cb6463a0 namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 78f66aa4-bcc3-45e5-b1a5-9df4d52ad26c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:08Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" in 16.905s (16.905s including waiting). Image size: 1355757128 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:08Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d06bb0b4d09 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 2f175806-c089-436d-b20d-5f1afd142c1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:08Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d06c105dd9a namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: a2fc2a14-2178-4518-89fc-0cd79ab815ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:08Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d06c1f8d461 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: ec48988b-7b32-41b6-b8a2-13bb234530b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d06c4c102d9 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: cd5b08b7-7a87-4fa9-82b2-e6c5def778cf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.963s (4.963s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:13Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d07ec9d06eb namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 363cfd11-042e-4f6b-9ec4-758ca895f41d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d07f12fc2b1 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: a62aa2d0-9587-4830-924f-195c859545ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d07f1c19cce namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 114b76b4-4df7-4449-9b7f-3f7eb2d1171e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d0853d8565d namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 0b424b38-4408-4409-aeb1-36f083bd50e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:20Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d17737dbd96 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 7b239074-eecc-4374-83ac-975a67c525e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:20Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d17737e52c9 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: a27b329a-ec9a-419a-ae24-6c383bbaa79f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b0855a28-11d1-4103-8d6f-8add0ecb15dd kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq.18d62d17b1fcdadc namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: f270e870-7357-4750-a442-3e0bd0e71c06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d46ec-predictor-7dbc68c665 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: f9466e13-102d-4889-a38b-9a55585929be kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'Created pod: error-404-isvc-d46ec-predictor-7dbc68c665-9zjhq' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: error-404-isvc-d46ec-predictor-7dbc68c665.18d62d028359cf6f namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: d073daa4-3357-4550-a559-28dced2edf5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: f6a5cc06-fc1a-4221-8dbc-e073eda9abaf kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: Scaled up replica set error-404-isvc-d46ec-predictor-7dbc68c665 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: error-404-isvc-d46ec-predictor.18d62d0280b5f566 namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: 01857d4b-e288-42bf-bfec-23ae745059a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 23386d4e-2370-4230-8d0b-4809e4f52a67 kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: error-404-isvc-d46ec-predictor.18d62d05ff79af8a namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 59ba1b16-e28d-44e8-b081-b8b5f8965810 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 23386d4e-2370-4230-8d0b-4809e4f52a67 kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: error-404-isvc-d46ec-predictor.18d62d0600651bf2 namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: a4405b3f-fee4-4e4b-9a14-bc55186dae2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 23386d4e-2370-4230-8d0b-4809e4f52a67 kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: error-404-isvc-d46ec-predictor.18d62d107ceee99f namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 51ef112d-6954-467b-b38b-96575bf54820 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 23386d4e-2370-4230-8d0b-4809e4f52a67 kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: error-404-isvc-d46ec-predictor.18d62d107d7b2605 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: d9ee8440-6c04-4b5d-92e6-94790103b225 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d46ec namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 9509e1f0-fe1d-4e0c-bb1f-e8b6be4cbb0a kind: Event lastTimestamp: "2026-09-17T17:51:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:01Z" name: error-404-isvc-d46ec.18d62d0280de0e76 namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 4fa8e830-bfcb-4e2b-bf0d-0873b244c16f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d46ec namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9509e1f0-fe1d-4e0c-bb1f-e8b6be4cbb0a kind: Event lastTimestamp: "2026-09-17T17:51:01Z" message: InferenceService [error-404-isvc-d46ec] is Ready metadata: creationTimestamp: "2026-09-17T17:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:01Z" name: error-404-isvc-d46ec.18d62d131052e1bf namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 12b3b907-c801-4d7e-ae0b-6cc80056fe79 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:11:01.107669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25149" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-de19c-predictor-d5c8fcc89-47k8v to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a4af8df56 namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 95b6a1cd-8823-4c4e-96a9-f9b0d0f50f31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25161" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a6ce4da92 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 71b315d8-e3af-4224-8748-588551889c59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a6e5cb9d5 namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 1d4f921b-cdf4-4c7b-a483-c26ccca004f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a713b9210 namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: fec67778-5f57-4615-81c9-97f57c68e8c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a71ab1118 namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: e723c7ef-e9ec-429c-b8eb-5f9a4452b798 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a71b671b1 namespace: kserve-ci-e2e-test resourceVersion: "25186" uid: d747cf47-43cf-43a0-8ced-77f8db946977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a74b4a214 namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 98427af3-fe53-4b76-931a-3767804650d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2a752ab16d namespace: kserve-ci-e2e-test resourceVersion: "25189" uid: bd92a6c3-efac-44ee-9530-8b775338fbee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:11:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:38Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e2ab1ea1215 namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: 2dfb133c-be71-451d-a924-bb3639cec5d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e3aa113b581 namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: d5d16734-3f19-4f6a-8a95-cc0702ccfaa9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e3aa114208c namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 82d7c216-351e-416c-8264-c60434589297 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: ce283b34-ac81-4d48-a2c1-a75f93b3958d kind: Event lastTimestamp: "2026-09-17T18:12:13Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:13Z" name: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v.18d62e3b3a0a0c21 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 5b3f1a43-d6a2-463c-a4e6-6cdc646e9abb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-de19c-predictor-d5c8fcc89 namespace: kserve-ci-e2e-test resourceVersion: "25142" uid: 1030232e-804d-41a6-9b7d-9de6b89aaf03 kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Created pod: error-404-isvc-de19c-predictor-d5c8fcc89-47k8v' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor-d5c8fcc89.18d62e2a4a6b2814 namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: 74377a64-2081-4cbb-ab4f-cd0157656d09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25140" uid: a495b57a-1e84-48ea-89d2-37f9838c05f5 kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Scaled up replica set error-404-isvc-de19c-predictor-d5c8fcc89 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c-predictor.18d62e2a487ea6aa namespace: kserve-ci-e2e-test resourceVersion: "25143" uid: 1ab30543-5686-47e2-a10e-09d87738cdf8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: 5c166d8b-174c-4e33-a6ee-f7ca050cf053 kind: Event lastTimestamp: "2026-09-17T18:11:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:46Z" name: error-404-isvc-de19c-predictor.18d62e2dc73c1073 namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: 8d0421cb-4a07-45ca-9ab5-53dfb84ec96e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: 5c166d8b-174c-4e33-a6ee-f7ca050cf053 kind: Event lastTimestamp: "2026-09-17T18:11:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:46Z" name: error-404-isvc-de19c-predictor.18d62e2dc82c3235 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: c9a743e1-c0f9-4975-9379-4ce56f60f1a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: 5c166d8b-174c-4e33-a6ee-f7ca050cf053 kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: error-404-isvc-de19c-predictor.18d62e3146e3d02c namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 2ae79f57-4ae4-44d3-9075-3d57546b709b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: 5c166d8b-174c-4e33-a6ee-f7ca050cf053 kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: error-404-isvc-de19c-predictor.18d62e3147534f08 namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: 29fe29f5-9d90-406b-8421-c8f5dcf05e6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: ca5b0af1-20ec-4355-ac12-e2a10be264c0 kind: Event lastTimestamp: "2026-09-17T18:11:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:48Z" name: error-404-isvc-de19c.18d62e2a48b23f9d namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 8872d055-1812-4b21-b5ae-dd5c1a88bbad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: ca5b0af1-20ec-4355-ac12-e2a10be264c0 kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-de19c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de19c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c.18d62e2a494fcaa5 namespace: kserve-ci-e2e-test resourceVersion: "25146" uid: bf3616c4-9b03-4c95-8276-f5eff0cf2cb2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: ca5b0af1-20ec-4355-ac12-e2a10be264c0 kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de19c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:01Z" name: error-404-isvc-de19c.18d62e2a495008d6 namespace: kserve-ci-e2e-test resourceVersion: "25148" uid: 4152324d-e8fe-4fad-b447-44715a527690 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: ca5b0af1-20ec-4355-ac12-e2a10be264c0 kind: Event lastTimestamp: "2026-09-17T18:11:48Z" message: InferenceService [error-404-isvc-de19c] is Ready metadata: creationTimestamp: "2026-09-17T18:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:48Z" name: error-404-isvc-de19c.18d62e356d91fa14 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: b1fa538c-3cbe-4acb-979a-a1ec4b7d883a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:12:11.444200Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e7d45-predictor-bfcffd599-ntx59 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:12:11Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3aab5b21b2 namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: 80a557bd-6860-409d-b3ec-d74aeb3611d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3acdb32970 namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: 649456d1-3eb5-4c0b-90c8-9bc280668ea7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3acf0058df namespace: kserve-ci-e2e-test resourceVersion: "26353" uid: da0a11dd-3800-4453-8d83-4d658dda4689 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3ad2155df1 namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: ad3bea55-1093-4513-9be3-a9c57e399c4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3ad28493a9 namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: fd17eb00-04f0-40b1-9415-9cbf8cbee453 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3ad28f0cb9 namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 15f240ad-2b20-4077-93a1-202f37b56a83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3ad60cf12f namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: de6f06f3-2843-43c4-8750-8446cd23c2e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3ad682b6d6 namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: b9263bc0-0497-4aaa-946f-c37b06227db4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T18:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:21:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:29Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62e3b109b41c3 namespace: kserve-ci-e2e-test resourceVersion: "30457" uid: d9fd5aee-c464-41e8-9009-7129229dd594 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:21:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:26Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62ebbd0afc88c namespace: kserve-ci-e2e-test resourceVersion: "30433" uid: aaf00e42-bfaa-48a9-bd16-1fb8e6fee28e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:21:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:26Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62ebbd0b0f4b2 namespace: kserve-ci-e2e-test resourceVersion: "30434" uid: d4f7f1ae-2d48-436f-96fe-fb955c3887e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 6fdf86e4-2b55-40a4-8ad2-385ee66aebc5 kind: Event lastTimestamp: "2026-09-17T18:21:29Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:29Z" name: error-404-isvc-e7d45-predictor-bfcffd599-ntx59.18d62ebc84fd4053 namespace: kserve-ci-e2e-test resourceVersion: "30456" uid: fad82850-94de-4607-8c28-e9251d2562b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e7d45-predictor-bfcffd599 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: aee46ec9-0ff5-45a9-a34a-9b065e2646c2 kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: 'Created pod: error-404-isvc-e7d45-predictor-bfcffd599-ntx59' metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:11Z" name: error-404-isvc-e7d45-predictor-bfcffd599.18d62e3aaaf9b849 namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: 78e10c4f-3d0f-43f4-9ff6-ee874f820678 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 6ee71896-0cfa-441d-b39c-f333721056ab kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Scaled up replica set error-404-isvc-e7d45-predictor-bfcffd599 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:11Z" name: error-404-isvc-e7d45-predictor.18d62e3aa9729b0c namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 740914c9-f0b7-44ea-82b4-740c07b17d38 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 0e4423fe-b8a7-423b-a6c0-7077700d873e kind: Event lastTimestamp: "2026-09-17T18:12:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:26Z" name: error-404-isvc-e7d45-predictor.18d62e3e27e8a8ba namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: 84ac0e48-9ad8-40da-928c-2e64a286ad2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 0e4423fe-b8a7-423b-a6c0-7077700d873e kind: Event lastTimestamp: "2026-09-17T18:12:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:26Z" name: error-404-isvc-e7d45-predictor.18d62e3e2846fdab namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 3476b94f-cc3d-46c5-aa6b-a2d34c4f3828 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 0e4423fe-b8a7-423b-a6c0-7077700d873e kind: Event lastTimestamp: "2026-09-17T18:13:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:26Z" name: error-404-isvc-e7d45-predictor.18d62e41a6ef2973 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: d2ea9781-7ddc-4063-8448-56112891dc93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 0e4423fe-b8a7-423b-a6c0-7077700d873e kind: Event lastTimestamp: "2026-09-17T18:13:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:26Z" name: error-404-isvc-e7d45-predictor.18d62e41a752d0bb namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: ab28dc0b-b8c0-40b6-9ed7-ac5ba1728fc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: e4399f73-9cf0-4fff-9024-6d2e77c0d66a kind: Event lastTimestamp: "2026-09-17T18:12:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:59Z" name: error-404-isvc-e7d45.18d62e3aa97bbc65 namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 2b4969d4-9fe5-4643-9c05-690a9371905a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26828" uid: e4399f73-9cf0-4fff-9024-6d2e77c0d66a kind: Event lastTimestamp: "2026-09-17T18:12:59Z" message: InferenceService [error-404-isvc-e7d45] is Ready metadata: creationTimestamp: "2026-09-17T18:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:59Z" name: error-404-isvc-e7d45.18d62e45cbffc44b namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: 9da27e74-8394-4490-a4c2-3956cf1025a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:50.472561Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:50Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d027337cdff namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 5300537d-6cdf-4294-83c2-a6a8ec9a748b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:50Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d027c15abb6 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 53b891bd-05f7-47ee-9b89-87032cb727a6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d02b9156bd2 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 51b54dfe-3359-44a2-be28-8af91991a55f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4352771c5cb4ba9624a857ef32395dc1abbb2576e467cd6caf52080736c17239" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d02bb6ba828 namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: e3c45f8e-e235-4d61-be42-7d6031e5d58e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d02bedc389c namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 42f5cf26-8406-4d25-b0db-d1b115b175d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d02bf53f46c namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: ed7aabb2-5c33-49a6-8e3b-3be80a9df9b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:49:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1970" metadata: creationTimestamp: "2026-09-17T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:56Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d03def5c134 namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 8ef8fc52-9e6c-4063-aecf-39f724717b13 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1970" in 12.577s (12.577s including waiting). Image size: 1580899218 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d06cca8c5d6 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: da726427-5cab-4f00-91c0-e104308c379e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d06d2784f63 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: c0b2c6d4-396f-4d4e-9c33-dd6afec7e5a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d06d3ca79fb namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 655c0824-075b-4102-bb7c-d37ed2ee8489 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d06d3d74526 namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: b2006f62-d2ff-4025-b593-0fe202448a23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.652s (4.652s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:13Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d07e923b03d namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 45ed1724-010a-442f-8aba-7822317230b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d07f0e27d0e namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 055f3004-233c-4e3d-865d-06d244048757 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d07f195eb09 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 4db19ab1-3fe1-4b29-b9ac-066a4cccfe50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:51:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:30Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d0817cf4c0b namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 9e1ab307-3437-45d3-954c-7e5c83726222 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d232fa18328 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: a8bd8353-417a-46c9-889b-ddb243c5116d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d232fa1ffb1 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: ec1dc558-8d52-49c5-997f-1c63eefc4822 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: e68679b4-02fe-4048-b466-dcb4dd11da7a kind: Event lastTimestamp: "2026-09-17T17:52:15Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:15Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x.18d62d24445eae99 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 589b806a-9265-4c80-b3cd-198db98d0800 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-74fc5cfbf9 namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 646b02cb-7698-43cc-8023-da55fe5f7c18 kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-74fc5cfbf9-4bx9x' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: isvc-sklearn-graph-1-predictor-74fc5cfbf9.18d62d0272c03d31 namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 8331daa0-8e06-4422-bf9d-2002ee4977a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 1ea5884e-008e-4f88-b6a4-5cb58a9755f0 kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-74fc5cfbf9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: isvc-sklearn-graph-1-predictor.18d62d026aaea7c4 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 16a28ae5-b491-495d-9802-4118fa2c30f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 68af192b-9d8c-4c7c-becf-4ab741a7bfbc kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: isvc-sklearn-graph-1-predictor.18d62d05f0d990dc namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 68a9814b-2d51-4899-92d7-7ed9d145d8a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 68af192b-9d8c-4c7c-becf-4ab741a7bfbc kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: isvc-sklearn-graph-1-predictor.18d62d05f1609161 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: ebdc67f9-4862-418f-b85a-f386d380863e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 68af192b-9d8c-4c7c-becf-4ab741a7bfbc kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: isvc-sklearn-graph-1-predictor.18d62d106eaf7a8c namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 63674abd-d0c0-4099-99fe-e2c45bade4a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 68af192b-9d8c-4c7c-becf-4ab741a7bfbc kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: isvc-sklearn-graph-1-predictor.18d62d106f16ccf4 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 4b8971d0-52c2-40fa-afd1-a54b4f2d962f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: de206c6b-defa-404b-9745-bb8739d4ad7e kind: Event lastTimestamp: "2026-09-17T17:51:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:37Z" name: isvc-sklearn-graph-1.18d62d0270e2f6ae namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: d5062af8-a67d-4efd-b6d4-4201d4d26e78 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: de206c6b-defa-404b-9745-bb8739d4ad7e kind: Event lastTimestamp: "2026-09-17T17:51:37Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-17T17:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:37Z" name: isvc-sklearn-graph-1.18d62d1b5744a9d4 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ee5b549e-e503-42c6-801f-1ac5ae1e3a2c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:51.069912Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d0296d29e18 namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 5bde3367-e1c2-4e64-84c2-ffc83e716892 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d02a0364f4a namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 93bb61df-5bfe-4ad1-a7a4-7ca6e6d280ef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:52Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d02dcc4c92b namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: 13530dd0-1d58-4071-9600-bea914061bfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4352771c5cb4ba9624a857ef32395dc1abbb2576e467cd6caf52080736c17239" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d02de42875e namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 24442ec8-fdac-4c16-a706-4bcf6659434d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d02e141fe3e namespace: kserve-ci-e2e-test resourceVersion: "14587" uid: fe448bb7-b5a6-46ec-9b59-dbf1c75ac953 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d02e1b5bf92 namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 515977af-f874-4988-a5a8-dfe724e0f994 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:49:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1970" metadata: creationTimestamp: "2026-09-17T17:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:57Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d041b8d4f96 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 05e98caa-aafc-461a-ad32-5775c92f038f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1970" in 11.482s (11.482s including waiting). Image size: 1580899218 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d06c7f06fc2 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 17f0932e-10b3-4756-af60-68faa5cd3de1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d06cd69a620 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: dd0f6075-374f-4c7c-acfe-b17e585e2e2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d06ce57a337 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: efa48f5e-c041-4162-bebf-ce821a8a307c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:09Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d06ce7cf26c namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: eaa16c7d-7d5f-42b2-953b-4d0bfa02e483 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.794s (4.794s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:13Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d07ec3cecb6 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 2834a533-e8b6-4458-a709-8b4f74783025 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d07f1088d3d namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: c84aa71f-243a-400d-b318-3e50ff7d7439 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:50:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:14Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d07f1b9448e namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 0ecd9b9e-8845-46f2-8530-b0d4f50dd42f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-17T17:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:52:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:12Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d0853da9b6b namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0f8324f5-9e36-438d-9bfb-fcfe7b511746 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d233360f1de namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: e5c4517d-5a17-461a-968e-81b5b6010482 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d23336191f9 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: bfff65ae-f066-4e16-ad7f-7273c41dc9c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 0130260a-d6c1-4b98-81d9-0ee4554b0a5a kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8.18d62d2356399243 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 24a2185f-3ac7-4fb7-981e-b7c03b197c33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-76857dd9b4 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 5243c560-2054-4b23-9cea-2cb5c21a94e2 kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-76857dd9b4-cvqt8' metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-2-predictor-76857dd9b4.18d62d02964ad684 namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: a94252d3-e57d-4abf-bcad-5edc785d47bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 44f00953-202b-4eb8-b9d4-c947f28cbc6d kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-76857dd9b4 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:51Z" name: isvc-sklearn-graph-2-predictor.18d62d029340bd5c namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 38438879-c6ff-4fe4-ae38-015593441b39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 53830f47-bf8e-4e6f-9b57-b15399045b74 kind: Event lastTimestamp: "2026-09-17T17:50:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:36Z" name: isvc-sklearn-graph-2-predictor.18d62d0612501a72 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: a5e722d0-9451-4b77-8abb-2630571322e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 53830f47-bf8e-4e6f-9b57-b15399045b74 kind: Event lastTimestamp: "2026-09-17T17:50:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:36Z" name: isvc-sklearn-graph-2-predictor.18d62d0612e0fdc5 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 29c9cbae-d551-4e28-9e22-2be9ba237fc0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 53830f47-bf8e-4e6f-9b57-b15399045b74 kind: Event lastTimestamp: "2026-09-17T17:52:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:06Z" name: isvc-sklearn-graph-2-predictor.18d62d108f0430fd namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: f1bce88a-ce7b-4e8c-a250-992946de2b24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 53830f47-bf8e-4e6f-9b57-b15399045b74 kind: Event lastTimestamp: "2026-09-17T17:52:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:06Z" name: isvc-sklearn-graph-2-predictor.18d62d108f586a2d namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 9b394638-ef45-4b47-9b2a-81c8b4acb984 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: c220002f-1b56-4624-b007-469475b900c2 kind: Event lastTimestamp: "2026-09-17T17:51:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:32Z" name: isvc-sklearn-graph-2.18d62d02934cb1f9 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 730ddb54-f172-42e1-a683-5c5a11168456 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: c220002f-1b56-4624-b007-469475b900c2 kind: Event lastTimestamp: "2026-09-17T17:51:32Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-17T17:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:32Z" name: isvc-sklearn-graph-2.18d62d1a2dc63598 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 223c5c4f-b618-4bcb-afe8-8206be60a94c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:50.878248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-xzhr5 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d028b660f71 namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 8aac9768-5fe1-423a-a091-3b0a3c919ce8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d02942601a2 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 0e8cee94-98e1-49e1-b659-ab79e8c6dd16 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d02d14d7a4f namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 6a812c91-f9bd-4696-ba86-7ae5609bf365 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4352771c5cb4ba9624a857ef32395dc1abbb2576e467cd6caf52080736c17239" already present on machine metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d02d3a04709 namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 9b86bbed-8960-4021-8a97-b80f3eb524b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d02d6766d2b namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 2c06a56c-117f-4ecb-8b02-3985af50e2c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d02d6e831ef namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 05688cae-545a-4d5a-81e0-7999d523c5c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:49:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-17T17:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d041b34fab7 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d1938da9-6459-48e7-a28c-bdd72ce40206 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 34.002s (34.002s including waiting). Image size: 1435742979 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c05e7e98a namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: d21353d2-cd62-4511-9ea1-26ce83afe2d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c0b1e441a namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 48cf7cc6-7a93-496a-b655-e87c784623a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c0c316fc5 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: a4d45d26-c2c1-4db2-871b-7e714ab65bb8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c0c4674cd namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 3f1ec4ed-2026-4ba8-bb97-63fecad64e38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c136dbd4e namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 09fd82be-d370-46da-a56a-c38322505d1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:50:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c14d91450 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 04dd00ae-e838-4ebf-9f20-c4b78327df84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-17T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:51:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d0c4e729808 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 752fe4f0-6138-4f93-96b6-a9d261c6dfcc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d2331487b5c namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 6ca9ce47-06ee-4e4e-ae4d-1ec5c1a407cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d233148f9fe namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 9cd4d1f8-4cec-4813-ab99-bd6f45fac571 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 496d8e39-534e-4318-be60-0f872782e0c4 kind: Event lastTimestamp: "2026-09-17T17:52:13Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-xzhr5.18d62d23d2af9ca6 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: b6cc0136-ba09-449d-922f-6ba3e3f0bc23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 7ffefa34-04f6-42ed-8d82-563b853f9151 kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-xzhr5' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d62d028af8f3e7 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 717d706e-258d-4666-9b05-65d50d9d6ad3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: b9e1bc18-879f-4e7f-97b2-c6d5b03b4c46 kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: isvc-xgboost-graph-predictor.18d62d028807f6e8 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 0e2012a4-9c6a-4ec1-bc78-7cc34d0e86ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: b422897a-b453-4580-8f51-2bcb5dd2babb kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: isvc-xgboost-graph-predictor.18d62d06077e6283 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: fdf4b57f-5036-451e-b75c-589f8f8db561 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: b422897a-b453-4580-8f51-2bcb5dd2babb kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: isvc-xgboost-graph-predictor.18d62d0609071dcf namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: b500d399-4267-410d-a5df-ef1900795331 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: b422897a-b453-4580-8f51-2bcb5dd2babb kind: Event lastTimestamp: "2026-09-17T17:52:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:06Z" name: isvc-xgboost-graph-predictor.18d62d1085780ef4 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: a1b39dba-f880-496c-a968-d15fdbfbd2e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: b422897a-b453-4580-8f51-2bcb5dd2babb kind: Event lastTimestamp: "2026-09-17T17:52:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:06Z" name: isvc-xgboost-graph-predictor.18d62d1085e5d33e namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 1caeb9e3-7d6e-4bf5-a5dd-800611fd6994 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: dfcdb42a-906a-4806-aa7f-296ab01e536d kind: Event lastTimestamp: "2026-09-17T17:51:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:39Z" name: isvc-xgboost-graph.18d62d02882ac3b8 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 8361bff8-a744-4a1a-b046-ba6fea5bf023 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: dfcdb42a-906a-4806-aa7f-296ab01e536d kind: Event lastTimestamp: "2026-09-17T17:51:38Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-17T17:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:38Z" name: isvc-xgboost-graph.18d62d1bb72017be namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 6784d771-58fd-4c24-9f49-abe58b92e4d7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:52:00.368738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-76bc4cf9d7-2h22s to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:52:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:52:00Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20b1a210cb namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 539b2cf4-b834-46dc-84a7-03d6609da4ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:00Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20b9c0ece8 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 015734b1-6a29-4678-a32c-06536fac1011 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:01Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:52:01Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20f7558f0d namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 01a0a19a-9807-4e1a-8dc0-409ceb50af65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:01Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20f8a9b8c0 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 579c2c50-9f9e-4663-8063-ff7479b19b49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:01Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-17T17:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:01Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20fbb8ac9d namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 7bedb515-b37f-460f-9105-3fb41cd10b19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:01Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-17T17:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:01Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d20fc34bc6c namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 794b61e0-34fb-4756-9702-4f4dbb473b72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:10Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-17T17:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:10Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d23280ecee3 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 88010af1-c029-4611-a5b0-c5c89605fffd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:52:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76bc4cf9d7-2h22s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 749b19de-b26d-4b84-97c3-a639fda6e829 kind: Event lastTimestamp: "2026-09-17T17:52:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:52:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:39Z" name: model-chainer-76bc4cf9d7-2h22s.18d62d23ebc7b02c namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: 2cc54ac5-7dcd-4e3f-aa12-e29eec5adc44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-76bc4cf9d7 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 14170985-ddcb-4bb9-931f-35da14580083 kind: Event lastTimestamp: "2026-09-17T17:52:00Z" message: 'Created pod: model-chainer-76bc4cf9d7-2h22s' metadata: creationTimestamp: "2026-09-17T17:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:00Z" name: model-chainer-76bc4cf9d7.18d62d20b11d9b92 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: ea781d8f-5450-4ff5-972d-a9411b82a6aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 766091bc-adde-4129-8513-6362de7542df kind: Event lastTimestamp: "2026-09-17T17:52:00Z" message: Scaled up replica set model-chainer-76bc4cf9d7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:00Z" name: model-chainer.18d62d20affa849d namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: c0bb37ab-efbf-4994-acb2-07b5c7f8e922 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 5e89ce02-05f7-483b-8e63-3a28e4d203fe kind: Event lastTimestamp: "2026-09-17T17:52:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-17T17:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:00Z" name: model-chainer.18d62d20b63d050d namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: e4d97d79-a3b2-438f-aa4b-eb87d1d02d45 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:53:11.156958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-081d8-66964d46f7-ghmwg to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d312cf09fd2 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 5e42abe4-72d8-42bc-aeb5-41cad706f9ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T17:53:11Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d314ef18301 namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: 86f277bf-f51f-4559-ad4f-664db57c2bfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T17:53:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d31508a3ddd namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: a3f8f6e7-c8f7-4b9b-b9e2-86c5cb08a396 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T17:53:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" in 1.841s (1.841s including waiting). Image size: 211751507 bytes.' metadata: creationTimestamp: "2026-09-17T17:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:53:13Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d31be4afcd5 namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: bf44c815-92ca-408a-89a8-7b4c66c6fc83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T17:53:13Z" message: 'Created container: sequence-graph-081d8' metadata: creationTimestamp: "2026-09-17T17:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:53:13Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d31c10dba85 namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: efe9367d-8f33-4a36-b503-02dc0c3dcc6c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T17:53:13Z" message: Started container sequence-graph-081d8 metadata: creationTimestamp: "2026-09-17T17:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:53:13Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62d31c184ad52 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: 35c9f627-aa0c-46fd-aadb-a0227b857de8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T18:01:25Z" message: Stopping container sequence-graph-081d8 metadata: creationTimestamp: "2026-09-17T18:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:25Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62da4575d1a42 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 65bb2198-f809-4732-828e-8c8d9c2acf88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-081d8} kind: Pod name: sequence-graph-081d8-66964d46f7-ghmwg namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 976221a4-d467-49c4-9b8b-174b8fba34e3 kind: Event lastTimestamp: "2026-09-17T18:01:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:55Z" name: sequence-graph-081d8-66964d46f7-ghmwg.18d62da570a9bb5b namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: bc15f77a-aeaa-402d-8c0c-f50fc5d91f67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-081d8-66964d46f7 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: 5bf83ece-e2b3-4004-aee6-1a7a51a84115 kind: Event lastTimestamp: "2026-09-17T17:53:11Z" message: 'Created pod: sequence-graph-081d8-66964d46f7-ghmwg' metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8-66964d46f7.18d62d312ca737ab namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 559f6358-f357-4cfb-afbf-ad787ca6b6e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5d6ab786-3add-43b7-90cc-28d0a45073f9 kind: Event lastTimestamp: "2026-09-17T17:53:11Z" message: Scaled up replica set sequence-graph-081d8-66964d46f7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8.18d62d312ba2ebea namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 3249acc9-1ad2-43f1-afa9-ed725075b9e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: a2ea985c-bce7-462b-83d6-683f533c53ac kind: Event lastTimestamp: "2026-09-17T17:53:11Z" message: InferenceGraph [sequence-graph-081d8] is Ready metadata: creationTimestamp: "2026-09-17T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:53:11Z" name: sequence-graph-081d8.18d62d31328ccb77 namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: a8f35540-c58d-465d-8131-21836adf18a9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: d8de170c-e740-44c1-9e79-01bcb16660dd kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: sequence-graph-081d8.18d62d34a9e327b4 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 2ebe87f9-dcbb-4b9e-b4fe-1b7b5485221c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: d8de170c-e740-44c1-9e79-01bcb16660dd kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: sequence-graph-081d8.18d62d34aa519b77 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: b30b1256-14b5-4708-a3c1-45f2b21c0225 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: d8de170c-e740-44c1-9e79-01bcb16660dd kind: Event lastTimestamp: "2026-09-17T17:53:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:56Z" name: sequence-graph-081d8.18d62d382925a7e2 namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: a9cf2384-f68d-4732-b614-a75d250f9b01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: d8de170c-e740-44c1-9e79-01bcb16660dd kind: Event lastTimestamp: "2026-09-17T17:53:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:56Z" name: sequence-graph-081d8.18d62d382999ee09 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 7f8c123e-86e3-4124-b5b1-0c90b0e733ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:02:26.084989Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8649f-fc945575-t7558 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:02:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:02:26Z" name: sequence-graph-8649f-fc945575-t7558.18d62db2613a60fa namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 0a6fd9c0-c63a-4e85-8ba4-9eed28772f40 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-8649f-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:26Z" name: sequence-graph-8649f-fc945575-t7558.18d62db26a906768 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: dfcb1174-45d2-4733-8914-d549e3b55049 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:27Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:02:27Z" name: sequence-graph-8649f-fc945575-t7558.18d62db2a6ae52cf namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 55e1dc28-fa06-4eda-8661-ffee4a9e3093 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8649f} kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:27Z" name: sequence-graph-8649f-fc945575-t7558.18d62db2a80066f4 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: dd952000-d0ec-4f63-ac2e-9d1d1c3c1ed8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8649f} kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:27Z" message: 'Created container: sequence-graph-8649f' metadata: creationTimestamp: "2026-09-17T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:27Z" name: sequence-graph-8649f-fc945575-t7558.18d62db2aadd3571 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: c241b175-c183-4998-83a7-3e8722d26edf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8649f} kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:27Z" message: Started container sequence-graph-8649f metadata: creationTimestamp: "2026-09-17T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:27Z" name: sequence-graph-8649f-fc945575-t7558.18d62db2ab44b3c3 namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: e73f39fd-7d1c-46f5-a5eb-55627e325c83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8649f} kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Stopping container sequence-graph-8649f metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: sequence-graph-8649f-fc945575-t7558.18d62db4b5181363 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 4bf8cd65-0ee5-4956-a9b8-8d6667d11334 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8649f} kind: Pod name: sequence-graph-8649f-fc945575-t7558 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 34575ee3-56a0-46aa-9055-7f1dbd70b55f kind: Event lastTimestamp: "2026-09-17T18:03:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:03Z" name: sequence-graph-8649f-fc945575-t7558.18d62db5537c7396 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 0f7d6716-0624-475f-9f91-ddf065c24c51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8649f-fc945575 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 187fe6b7-0e80-4835-9443-97974d615e2e kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'Created pod: sequence-graph-8649f-fc945575-t7558' metadata: creationTimestamp: "2026-09-17T18:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: sequence-graph-8649f-fc945575.18d62db2605b4951 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 3f96c803-1565-418b-8a47-4ee6b13a11b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8649f namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: c16967c9-1384-489e-8c12-fa79bbf2365c kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: Scaled up replica set sequence-graph-8649f-fc945575 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: sequence-graph-8649f.18d62db25d3f4300 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 52d37d96-632e-434f-921f-d3a6eb1509fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8649f namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 47fff3d8-3a8d-478c-b9a9-0ac0e68fa1d5 kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: InferenceGraph [sequence-graph-8649f] is Ready metadata: creationTimestamp: "2026-09-17T18:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:26Z" name: sequence-graph-8649f.18d62db2690d16e7 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 69a8c418-b096-4e14-b2ce-b285c0595537 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:03:36.349972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8eede-bdd8448d-m85hx to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62dc2bd590005 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 8d56b94e-ac8e-4ce6-80f5-3dd1fe064376 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62dc2df7555be namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: df2c9ec9-8fcb-45c0-b46a-1235f460159c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8eede} kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62dc2e0d87aeb namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 8bb1fa8e-c5b2-494a-b60a-d5a18084bf5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8eede} kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: 'Created container: sequence-graph-8eede' metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62dc2e3d59fd2 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 45dbe177-69ee-437c-9a6f-c72999ff34a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8eede} kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:03:37Z" message: Started container sequence-graph-8eede metadata: creationTimestamp: "2026-09-17T18:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:37Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62dc2e4464125 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 09024089-d9a3-4b70-a93e-c89c91e0aefd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8eede} kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:11:50Z" message: Stopping container sequence-graph-8eede metadata: creationTimestamp: "2026-09-17T18:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:50Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62e35e57d64e5 namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 963d4145-efcc-4d95-850e-6f1176006b3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8eede} kind: Pod name: sequence-graph-8eede-bdd8448d-m85hx namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: a4197f5d-92b4-44b7-a820-e3d30f8882a4 kind: Event lastTimestamp: "2026-09-17T18:12:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:18Z" name: sequence-graph-8eede-bdd8448d-m85hx.18d62e366204251e namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 592d3995-b074-46f5-b5f6-3223a98ba1ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8eede-bdd8448d namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 90e956af-8bd4-4573-9877-1b0525df5406 kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: 'Created pod: sequence-graph-8eede-bdd8448d-m85hx' metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede-bdd8448d.18d62dc2bcec9f21 namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 84172368-890c-432d-803b-5e2434147348 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8eede namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: fd06ec4d-9630-4126-8a5d-d6c843d42d0b kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: Scaled up replica set sequence-graph-8eede-bdd8448d from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede.18d62dc2bbd62686 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: f2010096-d3f6-496d-bcab-9b82dcb3621c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8eede namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: 28c6a85c-6439-4614-a605-6cb310b1c347 kind: Event lastTimestamp: "2026-09-17T18:03:36Z" message: InferenceGraph [sequence-graph-8eede] is Ready metadata: creationTimestamp: "2026-09-17T18:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:03:36Z" name: sequence-graph-8eede.18d62dc2c272e05f namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 3fc24adb-dac0-4a89-9c12-83fa6044b902 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:03:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8eede namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 1a4c8810-5045-41ca-8545-d5f7241e6b70 kind: Event lastTimestamp: "2026-09-17T18:04:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:04:06Z" name: sequence-graph-8eede.18d62dc63a553930 namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: e1871ed1-77d0-436f-97ab-9d72fa81599b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:03:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8eede namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 1a4c8810-5045-41ca-8545-d5f7241e6b70 kind: Event lastTimestamp: "2026-09-17T18:04:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:04:06Z" name: sequence-graph-8eede.18d62dc63ac51780 namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: e5874b8b-254e-4d31-8cc3-2a4a3619f31e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:12:01.050462Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-de19c-847bdf9fb9-z5b2d to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e383fd738fe namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 81552165-c98f-461c-b41c-9819101b851a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e3862805ec8 namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: 71623fb6-d6af-425e-8a0b-79ccc9499c1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-de19c} kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e3863f6ad12 namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: 82b35edb-d34c-4383-86a2-9adf2a4be977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-de19c} kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'Created container: splitter-graph-de19c' metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e3866c712c5 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: b9828519-554f-469a-bf12-bd8b9b5ff6a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-de19c} kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: Started container splitter-graph-de19c metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e386736afc3 namespace: kserve-ci-e2e-test resourceVersion: "26145" uid: 173e6bbd-c46c-4f70-b69b-c5483192a588 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-de19c} kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Stopping container splitter-graph-de19c metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e3a97f0ab25 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 0d2ea743-a3f5-4f12-80de-b443d3007df3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-de19c} kind: Pod name: splitter-graph-de19c-847bdf9fb9-z5b2d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 09c323ff-00a3-4cc8-963b-1030a490a9cd kind: Event lastTimestamp: "2026-09-17T18:12:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:38Z" name: splitter-graph-de19c-847bdf9fb9-z5b2d.18d62e3b0e0f6c3a namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 25859c47-f496-43b9-87ca-008bb3ed8fbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-de19c-847bdf9fb9 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: d42ff093-0355-4e85-835f-a5425bc0a96d kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'Created pod: splitter-graph-de19c-847bdf9fb9-z5b2d' metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c-847bdf9fb9.18d62e383f510c26 namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: 2f9cc8f6-cd67-4f7c-9a15-5ccafae0b493 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-de19c namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: 76be3bcb-0b9c-4a4f-b122-6581202234e1 kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: Scaled up replica set splitter-graph-de19c-847bdf9fb9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c.18d62e383e8fee95 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 4ba0663f-f4bf-45d1-acff-f42edff6cd3e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-de19c namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: ee36749c-f07d-4333-a35d-8270b391ae59 kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: InferenceGraph [splitter-graph-de19c] is Ready metadata: creationTimestamp: "2026-09-17T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:01Z" name: splitter-graph-de19c.18d62e3843debed0 namespace: kserve-ci-e2e-test resourceVersion: "26131" uid: 63525810-1c9f-46ad-9609-ef3d77675d9c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:13:11.368729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e7d45-7d4894f44c-9lvpm to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62e489f22fe22 namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: d94c3a74-eadc-4148-b182-f57d6db6f6b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:13:11Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62e48c10ef778 namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: 159448d1-1878-471e-9bf8-d76222caa2d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e7d45} kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:13:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62e48c28728d7 namespace: kserve-ci-e2e-test resourceVersion: "26971" uid: 480e9a18-cf85-4cef-9bc7-36e1888ccf1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e7d45} kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:13:12Z" message: 'Created container: splitter-graph-e7d45' metadata: creationTimestamp: "2026-09-17T18:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:13:12Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62e48c5bf2a80 namespace: kserve-ci-e2e-test resourceVersion: "26973" uid: b08c0a96-b261-4275-bd1a-46ee5d5b79cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e7d45} kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:13:12Z" message: Started container splitter-graph-e7d45 metadata: creationTimestamp: "2026-09-17T18:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:13:12Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62e48c63fd4a0 namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: 9aeb688c-cd0a-401e-b508-6d1bc16f303b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e7d45} kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:21:25Z" message: Stopping container splitter-graph-e7d45 metadata: creationTimestamp: "2026-09-17T18:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:25Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62ebbc7b990a3 namespace: kserve-ci-e2e-test resourceVersion: "30401" uid: ae2de1f3-6fef-432f-8bdb-12311d604196 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e7d45} kind: Pod name: splitter-graph-e7d45-7d4894f44c-9lvpm namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2e9f62b6-51ab-469c-8b94-3839312c1653 kind: Event lastTimestamp: "2026-09-17T18:21:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:53Z" name: splitter-graph-e7d45-7d4894f44c-9lvpm.18d62ebc58a60e12 namespace: kserve-ci-e2e-test resourceVersion: "30634" uid: fdad562a-3e65-4d1f-b20f-1ee567900234 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e7d45-7d4894f44c namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 3443d7e2-8840-4ef1-a27c-1f438991ebbb kind: Event lastTimestamp: "2026-09-17T18:13:11Z" message: 'Created pod: splitter-graph-e7d45-7d4894f44c-9lvpm' metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45-7d4894f44c.18d62e489eb09e9e namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: e17c2f93-7ccd-4b85-9fdf-840d34a6cf79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 4f3ce35d-aa32-4946-9b6f-45131820d4ba kind: Event lastTimestamp: "2026-09-17T18:13:11Z" message: Scaled up replica set splitter-graph-e7d45-7d4894f44c from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45.18d62e489dfbbba2 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 1bb13dbf-e1f7-4c3b-bc5d-6a062fbbc183 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: 221fef09-171c-4c66-a354-f983772c1f6e kind: Event lastTimestamp: "2026-09-17T18:13:11Z" message: InferenceGraph [splitter-graph-e7d45] is Ready metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:13:11Z" name: splitter-graph-e7d45.18d62e48a40770cb namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: 9b949d49-d895-4e34-8092-c2dfc1a885f3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 1e239ed1-7f25-43f0-be4e-5a77e38ef444 kind: Event lastTimestamp: "2026-09-17T18:13:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:26Z" name: splitter-graph-e7d45.18d62e4c1c6e1f39 namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 4a6b6fba-03ff-4e05-b9e5-a5140589674e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 1e239ed1-7f25-43f0-be4e-5a77e38ef444 kind: Event lastTimestamp: "2026-09-17T18:13:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:26Z" name: splitter-graph-e7d45.18d62e4c1ccc8925 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: a387613a-e016-4fdb-b614-0f3aa573ce48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: 1e239ed1-7f25-43f0-be4e-5a77e38ef444 kind: Event lastTimestamp: "2026-09-17T18:13:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:41Z" name: splitter-graph-e7d45.18d62e4f9b7fb0b0 namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: 6d21c59f-e856-4177-abd2-41282482051b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: 1e239ed1-7f25-43f0-be4e-5a77e38ef444 kind: Event lastTimestamp: "2026-09-17T18:13:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:41Z" name: splitter-graph-e7d45.18d62e4f9be1c3d5 namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: 8826dce0-43c3-4fc1-a54f-597ac74f7522 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:52:11.231519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-081d8-predictor-6b56b68db-qlfsp to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d23391b02e5 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 05f17dfd-4b9a-456f-a4de-9b8df8dd55e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d235b2b9dbd namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 7d52cf5e-f073-4e44-b8a3-77d8355b3f11 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d235cb1a373 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: fc12e413-46ab-45c9-bd7d-3c9a828724f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d235f65ed58 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 657e324b-3106-4bb1-b6b1-d03abb475b0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d235fdb9d80 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 2640c21f-0221-4205-936a-b8f280c52076 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d235fe7475a namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: a02e4f49-b8b9-46bf-96eb-f8ae64461c6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d2362a3a51b namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: a0b904f2-06f2-4155-beca-f3e4128ef2d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d236315b883 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 14eceae0-a20e-4ca7-a7ed-ebf8347a2aab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T18:01:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:28Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62d23766a6f08 namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 6f390c0b-7183-4ddd-95a1-eddbe1350c94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T18:01:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:25Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62da45ee3fdd4 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 25392f3d-e70f-4fb1-845a-9e0a51815501 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T18:01:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:25Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62da45ee490b1 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 3f4daa40-6dc2-41fa-aa09-230b5c5f14b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 9df9a881-bfd4-4352-8f03-d0060649fb9d kind: Event lastTimestamp: "2026-09-17T18:01:28Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:28Z" name: success-200-isvc-081d8-predictor-6b56b68db-qlfsp.18d62da4eac2b5bb namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 5ad705f5-61a1-40eb-aad9-9860352edfb7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-081d8-predictor-6b56b68db namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 03b1cf26-517e-4c2c-bf86-1870b6cddaad kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: 'Created pod: success-200-isvc-081d8-predictor-6b56b68db-qlfsp' metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor-6b56b68db.18d62d23389e58bd namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: ff24de9e-0564-42d3-8eed-74dd02b8bdcf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 0b35de46-c978-46fc-a952-cc758f802b23 kind: Event lastTimestamp: "2026-09-17T17:52:11Z" message: Scaled up replica set success-200-isvc-081d8-predictor-6b56b68db from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:11Z" name: success-200-isvc-081d8-predictor.18d62d2336ce0a4a namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: f61d9a99-4875-49b0-8d46-051f1cb54c94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: d2b56156-b78b-47d1-b963-d3654cf94a14 kind: Event lastTimestamp: "2026-09-17T17:52:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:26Z" name: success-200-isvc-081d8-predictor.18d62d26b55c2422 namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 5f914912-8c99-485a-ac55-69cdfdd5f6c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: d2b56156-b78b-47d1-b963-d3654cf94a14 kind: Event lastTimestamp: "2026-09-17T17:52:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:26Z" name: success-200-isvc-081d8-predictor.18d62d26b5ca375e namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: fe5df706-9a41-4dbc-974b-3e5efff9bd21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:52:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: d2b56156-b78b-47d1-b963-d3654cf94a14 kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: success-200-isvc-081d8-predictor.18d62d2a34cc6a20 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 68f0bfe7-d5c9-4e75-ae9c-1f2970d227c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:52:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-081d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: d2b56156-b78b-47d1-b963-d3654cf94a14 kind: Event lastTimestamp: "2026-09-17T17:53:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:53:26Z" name: success-200-isvc-081d8-predictor.18d62d2a359b89fc namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 0c7afb21-bdfd-4dad-8ea3-fd5a1df87071 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 4dfeb172-ac60-4c46-ab39-ab73843108b2 kind: Event lastTimestamp: "2026-09-17T17:52:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:58Z" name: success-200-isvc-081d8.18d62d2336e83a55 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: b39ed362-6acd-4d50-9588-f02f653f21ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-081d8 namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: 4dfeb172-ac60-4c46-ab39-ab73843108b2 kind: Event lastTimestamp: "2026-09-17T17:52:58Z" message: InferenceService [success-200-isvc-081d8] is Ready metadata: creationTimestamp: "2026-09-17T17:52:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:58Z" name: success-200-isvc-081d8.18d62d2e320c9540 namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: c2811ded-772e-4b4e-835c-bcfd2c6209f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:00:35.836810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:00:35Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98b5ecaa28 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: fff02c2f-19fb-41a8-b49a-b3eb7018a745 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98d7f6d04a namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 5a09bcd9-6fda-4ea8-84bd-ec7ed824ac3b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98d96193b4 namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: a8dacab8-e14c-4105-9959-234d0054589f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98dc3dd88c namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: 74e90c69-07a7-4584-a801-01c757bb095e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98dcb34e46 namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: f5850270-ba25-49a6-af09-35a93f9d3c41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98dcbe8242 namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 707031ab-e689-4d88-a3cd-beb0f903da6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98e047d003 namespace: kserve-ci-e2e-test resourceVersion: "19938" uid: 32f567d8-f5b4-4c66-98d9-b8632fe5e27a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:36Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d98e0c3e58f namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 21501902-bc0c-494f-9539-8fc150449fa5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:01:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:14Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62d9952f57a18 namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: 1090fc9e-36f4-460f-b088-34bd45671137 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62da90d59d551 namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 93c22127-32a6-40c5-b933-9522cd3c8ac9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 41e60a16-4e42-40b3-8f56-3af7a0ec10ed kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48.18d62da90d5a9013 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 2592a437-d934-4ac7-bcf7-e0b9d4e3e11a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0dcf6-predictor-5f6f74f858 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: ce988701-0311-452f-bb9d-eb0acb829775 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: 'Created pod: success-200-isvc-0dcf6-predictor-5f6f74f858-hls48' metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:35Z" name: success-200-isvc-0dcf6-predictor-5f6f74f858.18d62d98b5562bc3 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 91e77127-ae5a-44ad-a3cf-26c9695913e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: e635e25a-6eaf-46e9-a5ab-d0419a497451 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Scaled up replica set success-200-isvc-0dcf6-predictor-5f6f74f858 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:00:35Z" name: success-200-isvc-0dcf6-predictor.18d62d98b1d44034 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: c25464de-e6b8-48cb-99be-f62ede9bd394 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 53fa04a8-02e1-4172-8ba9-c4d643d9a974 kind: Event lastTimestamp: "2026-09-17T18:01:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:05Z" name: success-200-isvc-0dcf6-predictor.18d62d9c30eb877f namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: f75a5b84-5577-4a22-b3a3-ebae55c6857f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 53fa04a8-02e1-4172-8ba9-c4d643d9a974 kind: Event lastTimestamp: "2026-09-17T18:01:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:05Z" name: success-200-isvc-0dcf6-predictor.18d62d9c3157f9b1 namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 6bc294b0-784a-4e87-879a-35c10b213e87 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 53fa04a8-02e1-4172-8ba9-c4d643d9a974 kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: success-200-isvc-0dcf6-predictor.18d62da32f20c069 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 4c965404-7cc5-42c6-9136-3178b1cfb946 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dcf6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 53fa04a8-02e1-4172-8ba9-c4d643d9a974 kind: Event lastTimestamp: "2026-09-17T18:01:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:35Z" name: success-200-isvc-0dcf6-predictor.18d62da32f6b6455 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 0c86159e-5655-4e6f-aa68-64b56fa8180e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 312f5bdb-f1cb-498d-bb4c-382367dcfd3a kind: Event lastTimestamp: "2026-09-17T18:01:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:01:24Z" name: success-200-isvc-0dcf6.18d62d98b1eb786f namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: cc231baf-cad3-4a97-b264-4befe5ff618d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dcf6 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 312f5bdb-f1cb-498d-bb4c-382367dcfd3a kind: Event lastTimestamp: "2026-09-17T18:01:24Z" message: InferenceService [success-200-isvc-0dcf6] is Ready metadata: creationTimestamp: "2026-09-17T18:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:01:24Z" name: success-200-isvc-0dcf6.18d62da40f05ecdc namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 6def5324-18d5-492a-bfca-685f49c2ff5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:51:20.760784Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8592d-predictor-b4d845d84-vrdmm to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:51:20Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d1778d0b7e9 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 179bd2b9-c6fb-4427-9c02-ee892b835c77 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d179a942b2a namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 2b43dfd3-6327-442c-a6e6-a40eae912df0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d179c49d6fe namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 89d60152-e378-4366-b9fe-abbfb13ab697 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d179f049cec namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 5668683d-4176-4e78-a26c-b42538e90ee9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d179f75398f namespace: kserve-ci-e2e-test resourceVersion: "15557" uid: 967121af-5854-4985-a2aa-b17d24c12050 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d179f7f6d81 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: e7bcbc4a-f165-4378-aa2c-5d8c0830e0ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d17a2d3e074 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: 288c7334-1dfb-4520-bea9-6fb53f3a6dd3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d17a348da25 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: ef9b8ede-7731-4f72-a0b5-88eb140d2c90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T18:00:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:38Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d17c4a53972 namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 825589de-7d77-44e5-a5fe-032b3db69340 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:35Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d98af35c8e1 namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: 226bb25a-7b18-4259-9741-563f79ff906e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:35Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d98af366399 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 544fa368-8002-47cc-a460-294ef73f90f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 86ee9c65-b9ff-4d87-ab12-60bc80b5d9c4 kind: Event lastTimestamp: "2026-09-17T18:00:38Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:38Z" name: success-200-isvc-8592d-predictor-b4d845d84-vrdmm.18d62d99390ca76f namespace: kserve-ci-e2e-test resourceVersion: "19966" uid: ff6ef97b-16c0-4bfe-8420-c1021ad1c174 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8592d-predictor-b4d845d84 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: ebbc38a9-3c6a-4fbd-bb36-f32c6f7c6d9f kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: 'Created pod: success-200-isvc-8592d-predictor-b4d845d84-vrdmm' metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:20Z" name: success-200-isvc-8592d-predictor-b4d845d84.18d62d177827b28a namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 07bafdca-ff0f-4e44-8b02-55cba63b45c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 0aa9a2f0-5fff-45e5-839e-348f6b13407c kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Scaled up replica set success-200-isvc-8592d-predictor-b4d845d84 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:20Z" name: success-200-isvc-8592d-predictor.18d62d17767d65c8 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 23917e36-cecb-4156-83f5-491a35e697dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: eb33673e-8e53-4c90-80f5-609d41ba57cb kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: success-200-isvc-8592d-predictor.18d62d1af560398d namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: dd6ec793-14c4-45e6-87d6-c5fb7c71b24f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: eb33673e-8e53-4c90-80f5-609d41ba57cb kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:50Z" name: success-200-isvc-8592d-predictor.18d62d1af5f2fe42 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: d14f3df0-012b-427b-b045-1e151ba62f91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: eb33673e-8e53-4c90-80f5-609d41ba57cb kind: Event lastTimestamp: "2026-09-17T17:52:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:35Z" name: success-200-isvc-8592d-predictor.18d62d21f39872b3 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 6f9f96af-5f84-4ffb-af7d-d5c3e54965e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:52:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8592d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: eb33673e-8e53-4c90-80f5-609d41ba57cb kind: Event lastTimestamp: "2026-09-17T17:52:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:35Z" name: success-200-isvc-8592d-predictor.18d62d21f40e92d8 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e7cfc486-9686-41d9-90cc-136b553f9144 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8592d namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: b40162c6-b81a-44f5-88ba-afe8419b91ad kind: Event lastTimestamp: "2026-09-17T17:52:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:08Z" name: success-200-isvc-8592d.18d62d17768c1123 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 6c415cef-7d1f-4666-a2d1-47ce144f702b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8592d namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: b40162c6-b81a-44f5-88ba-afe8419b91ad kind: Event lastTimestamp: "2026-09-17T17:52:08Z" message: InferenceService [success-200-isvc-8592d] is Ready metadata: creationTimestamp: "2026-09-17T17:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:08Z" name: success-200-isvc-8592d.18d62d228a92811c namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 20c37908-d1b7-4077-9558-3267f422aa0b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:11:51.189118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e35f40f2334 namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 3406b9a7-fd9a-42e0-bfb7-8dbdf40b9b7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e3616693c56 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: 51399c86-0d63-4f43-9188-91fe0e8b9cd0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e3617e46387 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 01af9659-3f4f-49c2-ad7b-b1dfb2862d12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e361a853606 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: b55f511f-e04a-4f98-9def-d0d42295ce49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e361afeabf5 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: feb2a68f-b7a5-4be6-ab6a-2044f0469484 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e361b0ac2e0 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 0fb5436d-089a-4883-baee-08b15c99853b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e361dc7439e namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: d7300b7a-b6be-4d03-8b71-918b1edd7c51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e361e4153ae namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: d358e8b6-4d74-435a-bc37-0e0d48831f7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:12:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:29Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62e367a2f4e03 namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 0e43eae6-869c-4e64-a320-6e8e12d51011 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:29:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:10Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62f27e738a5ba namespace: kserve-ci-e2e-test resourceVersion: "33628" uid: 712d0d81-87ec-4618-8fcf-b4a0fccf9f7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 1f44c675-ab1a-4cc8-80d0-6bbebdcee50a kind: Event lastTimestamp: "2026-09-17T18:29:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:10Z" name: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg.18d62f27e7392153 namespace: kserve-ci-e2e-test resourceVersion: "33629" uid: 57cce174-7445-48b6-b277-4c379a7fa422 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-85bfa-predictor-74c5d488b8 namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 4caa42a4-0b9b-4886-8b18-fe9fca0150e4 kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: 'Created pod: success-200-isvc-85bfa-predictor-74c5d488b8-xfmfg' metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor-74c5d488b8.18d62e35f3481f01 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: b8ad7892-0498-49ed-bea7-f0616efa2d7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: da5e6991-8ec5-417f-8676-c2a397562e31 kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Scaled up replica set success-200-isvc-85bfa-predictor-74c5d488b8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-85bfa-predictor.18d62e35f18631b2 namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: f19572af-897e-4f3d-8d3c-62938027510d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: bbb96953-d62d-4fa1-be9b-a99507057291 kind: Event lastTimestamp: "2026-09-17T18:12:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:06Z" name: success-200-isvc-85bfa-predictor.18d62e39708439db namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: ed17950a-c34b-462c-9463-3222e19902b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: bbb96953-d62d-4fa1-be9b-a99507057291 kind: Event lastTimestamp: "2026-09-17T18:12:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:06Z" name: success-200-isvc-85bfa-predictor.18d62e3971026801 namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 7dd508a5-4f9d-43cc-8820-c4fbf7f4c91c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: bbb96953-d62d-4fa1-be9b-a99507057291 kind: Event lastTimestamp: "2026-09-17T18:13:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:06Z" name: success-200-isvc-85bfa-predictor.18d62e3cf09dc392 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: a4dd7809-c38b-4fb8-86e4-87305973e8a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85bfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: bbb96953-d62d-4fa1-be9b-a99507057291 kind: Event lastTimestamp: "2026-09-17T18:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:06Z" name: success-200-isvc-85bfa-predictor.18d62e3cf123bba3 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: 713a38fa-e80f-43aa-8541-d4369be7f112 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85bfa namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 4d17fd8a-c8ec-4ab3-af34-c92692d62f39 kind: Event lastTimestamp: "2026-09-17T18:12:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:39Z" name: success-200-isvc-85bfa.18d62e35f1b91550 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: b8bcbb14-9ce7-4905-bdc1-f93b19ce574f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: 4d17fd8a-c8ec-4ab3-af34-c92692d62f39 kind: Event lastTimestamp: "2026-09-17T18:12:39Z" message: InferenceService [success-200-isvc-85bfa] is Ready metadata: creationTimestamp: "2026-09-17T18:12:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:39Z" name: success-200-isvc-85bfa.18d62e4135b520aa namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 53c1f049-3e87-4470-8f49-84b51ef9354a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:01:26.062971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:01:26Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da467a317ab namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: f1a27854-fb06-429c-bdd3-af979cc1a49b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8649f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:26Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4721ddb1a namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: edaaf14e-3b07-48cd-91fc-97f2facecc86 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4ad6cf00c namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: abdf5129-9692-4628-8baf-6a6278aa4732 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4aed41eb8 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 4abc2fde-00bb-43fd-aa52-d3cfc43ab5c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4b1c11941 namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: d2ddbf70-03f3-40d0-a7a3-5618fa4cffab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4b23823ad namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 37c527b6-4bf2-4f4a-907b-feee0738ecd3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4b24220d6 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 2db602bd-4f63-413f-9c5d-7bf7f2a4d299 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4b5d9ee08 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 00556712-36e2-4f90-9098-4cc8d8de235d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:01:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:27Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4b650dccd namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: c0f10b7e-001d-4a14-a1c7-16c007e67197 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:02:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:03Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62da4c8a2ba36 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 170d6107-b0a2-4130-a7ab-ec0d7330de13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62db4bb0d4553 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: ee1e24a0-663e-4d67-942e-d0147f7133fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62db4bb0dc7d9 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 34a14bfa-944c-4d99-a42a-b85b54b16a5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 0aa7a903-cc6e-45d4-8b34-eb93a0d0cab3 kind: Event lastTimestamp: "2026-09-17T18:02:38Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:38Z" name: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr.18d62db550d13f4d namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 2c07256b-8254-4f03-888e-98e9ec27f0a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8649f-predictor-86d8f8bc54 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 8588c487-79e9-4b8f-961f-6120c858bb6e kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: 'Created pod: success-200-isvc-8649f-predictor-86d8f8bc54-9q6dr' metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:26Z" name: success-200-isvc-8649f-predictor-86d8f8bc54.18d62da466e585d4 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: bd24be83-d356-4908-9c79-7add58497269 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: bdaadde0-3c94-46c4-abaa-0ce2638ac080 kind: Event lastTimestamp: "2026-09-17T18:01:26Z" message: Scaled up replica set success-200-isvc-8649f-predictor-86d8f8bc54 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:26Z" name: success-200-isvc-8649f-predictor.18d62da464ddb027 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: bd8c165b-bed3-49f9-8356-ca1a2d4b12e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: e204a116-0f8e-48b6-ad02-527f16dff24d kind: Event lastTimestamp: "2026-09-17T18:02:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:11Z" name: success-200-isvc-8649f-predictor.18d62da7e3bb6b4a namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: c27725cd-cbc7-435e-8def-7f5ca45b78f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: e204a116-0f8e-48b6-ad02-527f16dff24d kind: Event lastTimestamp: "2026-09-17T18:02:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:11Z" name: success-200-isvc-8649f-predictor.18d62da7e47dec23 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 5a926104-5d21-4635-951f-64049cc16fc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: e204a116-0f8e-48b6-ad02-527f16dff24d kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: success-200-isvc-8649f-predictor.18d62dab63422ca9 namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 4f816718-c011-4a54-9be2-84bc11850dff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:01:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8649f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: e204a116-0f8e-48b6-ad02-527f16dff24d kind: Event lastTimestamp: "2026-09-17T18:02:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:26Z" name: success-200-isvc-8649f-predictor.18d62dab63abdd19 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: cc36edc6-2dca-4ee3-82f8-d2d0a02f7f81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8649f namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: b531835e-5021-4bdb-af2d-abb9d9f43951 kind: Event lastTimestamp: "2026-09-17T18:02:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:13Z" name: success-200-isvc-8649f.18d62da464f1f35f namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 360f131e-6d5e-448a-88eb-00e67435ab27 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8649f namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: b531835e-5021-4bdb-af2d-abb9d9f43951 kind: Event lastTimestamp: "2026-09-17T18:02:13Z" message: InferenceService [success-200-isvc-8649f] is Ready metadata: creationTimestamp: "2026-09-17T18:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:13Z" name: success-200-isvc-8649f.18d62daf84b7a56a namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 08ffb605-c6ce-42e7-97a9-4c8e33ea1797 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:02:36.309011Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8eede-predictor-757747b4bb-fm7b8 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db4c2a0a2ef namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 14a7f60c-4c74-4881-bfb3-834d9b2b65bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8eede-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db4cf60c90f namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 63fba0ff-7b96-4804-8e0e-ef7a9c7b7b85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db508b592c7 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: aa2e77bb-3586-4e3a-ac43-5fbd33e26d85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db50ac15e4b namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: ef39d5c2-7593-4fa9-aafd-03757d1d2181 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db50e24928c namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 6f02c3c5-2e5a-4839-8dc5-e90e2023e2fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db50eb143a0 namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: 8dbfa5b3-b89c-4e80-bf59-9bf7b1a6b290 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db50ebd5c86 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 07208951-0706-4fb6-b0d3-ae5aa29feabd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db51258584b namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: d34a320e-0546-4c58-bcc9-dc2a14157fe9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:02:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:37Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db512d702b1 namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 538eea3f-ae39-4d4f-b3cf-1d38d58fb377 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:03:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:03:15Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62db563ef2429 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 672713b4-2f71-4ed9-ae01-263feedd8edb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62e35ebfcba48 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: 90385be3-9ae8-44a2-97c0-32c839952a19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 9bf80e37-d9f6-4ce8-8666-14bdd0b681ce kind: Event lastTimestamp: "2026-09-17T18:11:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:51Z" name: success-200-isvc-8eede-predictor-757747b4bb-fm7b8.18d62e35ebfd38c3 namespace: kserve-ci-e2e-test resourceVersion: "25920" uid: 3c5a77ae-2d9a-466f-8789-32e17817300f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8eede-predictor-757747b4bb namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 1262da2c-5a4b-4089-a108-548eb4fda831 kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: 'Created pod: success-200-isvc-8eede-predictor-757747b4bb-fm7b8' metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8eede-predictor-757747b4bb.18d62db4c2790606 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: dc06a1a2-b3ef-46fe-8023-c2ae449d6723 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: ed23f799-181c-4e6f-ab15-b820efa3d69f kind: Event lastTimestamp: "2026-09-17T18:02:36Z" message: Scaled up replica set success-200-isvc-8eede-predictor-757747b4bb from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:36Z" name: success-200-isvc-8eede-predictor.18d62db4c0820930 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: e7b058a9-49a3-4146-aae4-23faaf588df7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: e8426462-97b8-4b35-808d-6665a81a5973 kind: Event lastTimestamp: "2026-09-17T18:03:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:06Z" name: success-200-isvc-8eede-predictor.18d62db83f44ee21 namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 0dfa8709-af6b-4844-b4e8-1bb88cbcdfc1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: e8426462-97b8-4b35-808d-6665a81a5973 kind: Event lastTimestamp: "2026-09-17T18:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:06Z" name: success-200-isvc-8eede-predictor.18d62db8404271c0 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: 5a0a7567-f296-48c1-acec-18e7a9ebb330 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: e8426462-97b8-4b35-808d-6665a81a5973 kind: Event lastTimestamp: "2026-09-17T18:03:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:51Z" name: success-200-isvc-8eede-predictor.18d62dbf3e5cc94e namespace: kserve-ci-e2e-test resourceVersion: "22145" uid: 73155104-c904-47c2-8b22-24879d0a6abb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8eede-predictor namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: e8426462-97b8-4b35-808d-6665a81a5973 kind: Event lastTimestamp: "2026-09-17T18:03:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:51Z" name: success-200-isvc-8eede-predictor.18d62dbf3eeaa10c namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: f5411ae2-59e8-4b7f-92e8-782f0c387f9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:02:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8eede namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: cbcb55f4-4429-436a-a4c0-a1bf2ccdd7ee kind: Event lastTimestamp: "2026-09-17T18:03:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:03:25Z" name: success-200-isvc-8eede.18d62db4c08bc5b7 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 3c88381e-c7c9-4d07-8a08-993cf8608531 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8eede namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: cbcb55f4-4429-436a-a4c0-a1bf2ccdd7ee kind: Event lastTimestamp: "2026-09-17T18:03:25Z" message: InferenceService [success-200-isvc-8eede] is Ready metadata: creationTimestamp: "2026-09-17T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:03:25Z" name: success-200-isvc-8eede.18d62dc02063ee5f namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 5cc97431-8fe2-4278-8f85-91e633dc514c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:01:46.145532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9d702-predictor-84c754796-qj7zl to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da914a69fb2 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 870452c1-effa-430e-8a79-738812baa7d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9d702-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da91e2321cb namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 2343d34e-b4af-4ba7-90b9-fd356a1ad488 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da95a597e92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: fa28bb5a-608a-469a-95dd-4d654bc79aa4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da95bddb5de namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 8260a479-f356-4736-bf9b-c14561d4f2a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da95fe96afc namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 00117cef-9ab5-42f5-b1d5-e21b0042e53f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da9605b8279 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 46dd2d46-4d3f-477d-bea6-8334dd6637a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da960665a52 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: f2acc98f-cf2f-4b79-8e4f-564fa9dfdca4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da963b13efd namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 6f820b90-ce6c-4f50-aa64-a57d62798535 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:01:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:47Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da964224346 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 9a1236f3-04f8-4418-ab17-2b6165f8e761 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:02:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:02:24Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62da9b2434775 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 064c65d0-46a7-45d7-b1a1-d0fd6672aa6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62e2a3ea67dfa namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: ca10b984-0ac1-47f3-8a58-a605001d8267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d702-predictor-84c754796-qj7zl namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 0235e986-f167-4b11-b51a-65e57f101ae1 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-9d702-predictor-84c754796-qj7zl.18d62e2a3ea708f1 namespace: kserve-ci-e2e-test resourceVersion: "25097" uid: d39053f3-c857-4faa-ad43-2f8907feb6b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9d702-predictor-84c754796 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 155b84f0-8a00-47cb-903b-2b51dd6b5233 kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: 'Created pod: success-200-isvc-9d702-predictor-84c754796-qj7zl' metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-9d702-predictor-84c754796.18d62da914385a5a namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 83696c7e-0ea1-4c48-b25e-e5d856e736ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 51185809-c698-4e82-a805-31f377d7e0e6 kind: Event lastTimestamp: "2026-09-17T18:01:46Z" message: Scaled up replica set success-200-isvc-9d702-predictor-84c754796 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:01:46Z" name: success-200-isvc-9d702-predictor.18d62da912878c5b namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 3f2ce5bc-d390-42eb-9512-252b054946ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ee34e6f2-3a94-427a-94e7-0c841f0ec31a kind: Event lastTimestamp: "2026-09-17T18:02:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:31Z" name: success-200-isvc-9d702-predictor.18d62dac911a4936 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: c44172bf-d8f5-40a1-b466-e1bec3fa7e41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:02:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ee34e6f2-3a94-427a-94e7-0c841f0ec31a kind: Event lastTimestamp: "2026-09-17T18:02:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:02:31Z" name: success-200-isvc-9d702-predictor.18d62dac9187ec88 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 768824de-19ea-4b9b-b458-19bdc3d113e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:02:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: ee34e6f2-3a94-427a-94e7-0c841f0ec31a kind: Event lastTimestamp: "2026-09-17T18:03:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:16Z" name: success-200-isvc-9d702-predictor.18d62db0109397bf namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 98e6b8cf-8106-4961-be8c-9d4f51c96493 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:02:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d702-predictor namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: ee34e6f2-3a94-427a-94e7-0c841f0ec31a kind: Event lastTimestamp: "2026-09-17T18:03:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:02:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:03:16Z" name: success-200-isvc-9d702-predictor.18d62db010fc3f87 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: e24d1572-9061-4f1a-a56b-0a64a649d392 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:01:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d702 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: b7d275ea-2fe5-4aec-a831-4d11181e6984 kind: Event lastTimestamp: "2026-09-17T18:02:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:34Z" name: success-200-isvc-9d702.18d62da9129e4e31 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 4494a634-d649-4341-90a8-f1a3f287d8df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:02:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d702 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: b7d275ea-2fe5-4aec-a831-4d11181e6984 kind: Event lastTimestamp: "2026-09-17T18:02:34Z" message: InferenceService [success-200-isvc-9d702] is Ready metadata: creationTimestamp: "2026-09-17T18:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:02:34Z" name: success-200-isvc-9d702.18d62db471c18cfe namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 1e763cde-fcc6-447e-9367-e7e959076ca6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:49:50.573444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:49:50Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d02793b3db7 namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 63040987-30f9-4554-b9ae-2edc571d0e4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:49:51Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d029b163cd6 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: a84e53ac-39f7-4064-ab98-cf4bedbc8382 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:49:51Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" metadata: creationTimestamp: "2026-09-17T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:49:51Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d029cdfe6a5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 9525efe4-ad40-4f5b-af16-e99246855027 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" in 11.934s (11.934s including waiting). Image size: 1356815952 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:03Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05643f05e6 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 46fc8bc1-f4c1-490b-aeb4-95401df4dc43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:03Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05672da364 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: a9a63560-6bce-4a88-b804-d55591d92cf6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:03Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d0567a063f4 namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 53d15666-927b-4ced-b028-2a53a4a0f578 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:03Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d0567afdf2f namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 38298ac3-0c2f-4689-b677-0372270f0c00 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:05Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.247s (2.247s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:05Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05eda12ab1 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 2bfac10c-e108-48be-8d41-717fec51a1b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:05Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05f0a8bc17 namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 01b9ad69-15b7-441f-afe1-23ad713b4b65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:50:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:50:05Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05f128e924 namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: 75e027d7-32ae-482d-8d57-8acc89bb52f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d05ff3f2aba namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 22064e1a-873c-49b7-a504-14c74a69fcdd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:20Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d1771b30f92 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 79850ab5-d1e6-472c-a773-b8b69c98fb70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:20Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d1771b3fe28 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 42a8b0ec-4ba1-45ca-a18e-112ff6e69332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: ba34d9f1-3398-473e-8112-96a472348d6e kind: Event lastTimestamp: "2026-09-17T17:51:21Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:21Z" name: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7.18d62d17b1634979 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: f8d96809-fe34-401f-8507-565436f71845 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d46ec-predictor-54ff8f5568 namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 4d97177f-9047-4364-8f34-97af63e106fb kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: 'Created pod: success-200-isvc-d46ec-predictor-54ff8f5568-kj6n7' metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: success-200-isvc-d46ec-predictor-54ff8f5568.18d62d02789add44 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 0119fd0e-27e4-49a4-ae37-6d91f176b85c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: 5d87d39c-c901-42d7-8099-032953430770 kind: Event lastTimestamp: "2026-09-17T17:49:50Z" message: Scaled up replica set success-200-isvc-d46ec-predictor-54ff8f5568 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:49:50Z" name: success-200-isvc-d46ec-predictor.18d62d0276d20832 namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 0af8cda8-10b9-4312-918c-bde71a727eb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 32803622-cf46-44bc-9c3e-f37136ef364c kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: success-200-isvc-d46ec-predictor.18d62d05f5e6ba28 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 9a416516-2ed5-4878-8d67-28ef46e4c901 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 32803622-cf46-44bc-9c3e-f37136ef364c kind: Event lastTimestamp: "2026-09-17T17:50:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:50:35Z" name: success-200-isvc-d46ec-predictor.18d62d05f67e3d6b namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 6d4be6f3-5a11-45ca-afe3-3edc1cc3e049 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 32803622-cf46-44bc-9c3e-f37136ef364c kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: success-200-isvc-d46ec-predictor.18d62d1072e02b62 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: ca0eaa28-df56-4215-926e-3c25b927c348 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:50:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d46ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 32803622-cf46-44bc-9c3e-f37136ef364c kind: Event lastTimestamp: "2026-09-17T17:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:05Z" name: success-200-isvc-d46ec-predictor.18d62d107356be88 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: f1a04973-7b90-496d-87a5-520e59a16445 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d46ec namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 4c5eff8c-3dfe-4a9c-a94b-5dbdf1a55fb4 kind: Event lastTimestamp: "2026-09-17T17:50:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:51Z" name: success-200-isvc-d46ec.18d62d027764f152 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 8c636b08-5c61-4418-bdef-f4319a328c05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:50:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d46ec namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: 4c5eff8c-3dfe-4a9c-a94b-5dbdf1a55fb4 kind: Event lastTimestamp: "2026-09-17T17:50:51Z" message: InferenceService [success-200-isvc-d46ec] is Ready metadata: creationTimestamp: "2026-09-17T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:50:51Z" name: success-200-isvc-d46ec.18d62d10bb19f150 namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 46a6b52f-1deb-49de-8bc4-6b16b623ea11 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:11:01.006257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a44ed9ba6 namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: d0928c20-3b12-497d-934e-d4dcc891e527 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25136" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a670b4b05 namespace: kserve-ci-e2e-test resourceVersion: "25171" uid: b40d578e-2272-4bf2-9866-76b475a7c184 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a688b2310 namespace: kserve-ci-e2e-test resourceVersion: "25174" uid: d92a0520-bf02-4a62-ad71-208e62bbe100 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a6b3359ee namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: 5a64a78c-ec16-434a-8cc0-1dc317c1e5d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a6ba638c1 namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 79d165a0-b5de-4cf8-a693-5648d82cf8ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a6bb1bb18 namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 3f8db7f9-3a49-49bc-8808-4573657d5b99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a6e6560f5 namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: 259ecd77-3653-43bc-a83f-2780b7caa197 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:01Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2a6edf1da6 namespace: kserve-ci-e2e-test resourceVersion: "25183" uid: b0b65215-9332-40cd-b49e-7dc5edc952c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:11:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:11:39Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e2ac8740d79 namespace: kserve-ci-e2e-test resourceVersion: "25756" uid: 192e8b2c-bf1c-4596-8a73-5dcdebc59a35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e3a9f14d8d2 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 912a133a-d10d-48d9-91b1-b3fbfe8c21e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e3a9f157737 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 4e145158-ec55-4934-abe9-dcefe9676384 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 2c615fa2-4831-444f-a916-0dfd927210ed kind: Event lastTimestamp: "2026-09-17T18:12:14Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:14Z" name: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2.18d62e3b50b5173e namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 13f9535f-cd84-4bd5-b172-f52172fad66c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-de19c-predictor-548b8d5b44 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 02563907-d94a-46a6-936b-dc705d2dc12a kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: 'Created pod: success-200-isvc-de19c-predictor-548b8d5b44-6g7k2' metadata: creationTimestamp: "2026-09-17T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-de19c-predictor-548b8d5b44.18d62e2a4475d48e namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: 5a7334b6-23b4-4ac1-a1e8-d4ba1eb25f55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: 598ae11c-28bf-4bf1-a731-b1fec3abdaa6 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: Scaled up replica set success-200-isvc-de19c-predictor-548b8d5b44 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-de19c-predictor.18d62e2a402e4b7d namespace: kserve-ci-e2e-test resourceVersion: "25102" uid: a574b2ae-7a87-445f-a03a-7d47be284495 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: da2a679b-cbde-449f-8b5a-020497f2329b kind: Event lastTimestamp: "2026-09-17T18:11:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:45Z" name: success-200-isvc-de19c-predictor.18d62e2dbf44c207 namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: 5d9ec55c-5da2-4e24-9aae-03dd5bae06be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: da2a679b-cbde-449f-8b5a-020497f2329b kind: Event lastTimestamp: "2026-09-17T18:11:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:11:45Z" name: success-200-isvc-de19c-predictor.18d62e2dbfc15ce3 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: 683b9d9b-a3c1-4707-91a2-a90ee749f64a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: da2a679b-cbde-449f-8b5a-020497f2329b kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: success-200-isvc-de19c-predictor.18d62e313e80bfcc namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 054e4361-7e78-4e66-99c0-ef5c06a3c8db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de19c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: da2a679b-cbde-449f-8b5a-020497f2329b kind: Event lastTimestamp: "2026-09-17T18:12:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:01Z" name: success-200-isvc-de19c-predictor.18d62e313ef1452a namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: 9ad3c7f9-dc29-498f-be1e-eef1b38f42f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 70077da9-cf32-4fae-8e19-d40ef88a8b04 kind: Event lastTimestamp: "2026-09-17T18:11:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:49Z" name: success-200-isvc-de19c.18d62e2a404a3c83 namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: 96c2a0b5-0251-422f-a876-19ddf8cd838a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 70077da9-cf32-4fae-8e19-d40ef88a8b04 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-de19c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de19c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-de19c.18d62e2a40fca7c3 namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 6bd55357-3f88-4f73-8457-fa5e400d3ab3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 70077da9-cf32-4fae-8e19-d40ef88a8b04 kind: Event lastTimestamp: "2026-09-17T18:11:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de19c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:00Z" name: success-200-isvc-de19c.18d62e2a40fcf3f7 namespace: kserve-ci-e2e-test resourceVersion: "25112" uid: 4dcb0216-f817-472c-b19e-2180f069bcf8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:11:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de19c namespace: kserve-ci-e2e-test resourceVersion: "25862" uid: 70077da9-cf32-4fae-8e19-d40ef88a8b04 kind: Event lastTimestamp: "2026-09-17T18:11:49Z" message: InferenceService [success-200-isvc-de19c] is Ready metadata: creationTimestamp: "2026-09-17T18:11:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:11:49Z" name: success-200-isvc-de19c.18d62e358462d963 namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: a7c1c75f-1d37-4956-a8dc-5c57f77b3a8d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:12:11.350957Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3aa5cc5c21 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 6eabf526-c659-4db9-a725-404b96569848 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e7d45-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3ab08fe1ed namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: 4b6d07c6-fe70-42a0-bf39-04dc640cf368 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3aeb916928 namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 350e1969-6239-4742-a5e4-5907612e9f80 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1970" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3aed12c6fa namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: ae435716-0364-4bb0-8a06-beafb7b7e227 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3af018a024 namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: a0778db3-4ae1-493e-a9a8-88ae939eb937 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3af09e86eb namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: baaeb86e-84c4-4245-92c7-c9c5813676b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3af0a90cff namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 03b38715-e145-4f75-bbba-e70e44b02159 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3af3d77033 namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: c1a2982f-e103-451e-9bfe-719eb91014c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:12Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3af44a6807 namespace: kserve-ci-e2e-test resourceVersion: "26372" uid: 849f0ab2-0c5d-4e3e-b405-10bcc6319c9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:12:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:49Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62e3b279c32fe namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 337e7d8b-c422-49de-a54c-466bc20660ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:21:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:26Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62ebbcedafc9c namespace: kserve-ci-e2e-test resourceVersion: "30419" uid: f21cd364-dd1c-45b4-bd3e-906c36c9838b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9f7c7a86-c19a-4817-972e-05b890b9e0ed kind: Event lastTimestamp: "2026-09-17T18:21:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:21:26Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss.18d62ebbceddaf5a namespace: kserve-ci-e2e-test resourceVersion: "30425" uid: f23dc1f9-7fa0-4979-9870-ddaccb227ae5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e7d45-predictor-685f9d9bbb namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: 2944732e-f917-4363-867f-61154df3a2fe kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: 'Created pod: success-200-isvc-e7d45-predictor-685f9d9bbb-mm5ss' metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-e7d45-predictor-685f9d9bbb.18d62e3aa566ec6a namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: bceb6b9f-220a-4ff9-a96f-37e45f282ba4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 181e6261-0ed5-4033-968e-c28cb73d6e6c kind: Event lastTimestamp: "2026-09-17T18:12:11Z" message: Scaled up replica set success-200-isvc-e7d45-predictor-685f9d9bbb from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:11Z" name: success-200-isvc-e7d45-predictor.18d62e3aa3dd7b89 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: a83e9550-e907-4288-b52b-8388c8fb86d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 38775a2e-de07-4079-a7b2-679ee2c84172 kind: Event lastTimestamp: "2026-09-17T18:12:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:56Z" name: success-200-isvc-e7d45-predictor.18d62e3e22625b9c namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 43e31434-e110-424a-b9b1-b61af51c6471 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 38775a2e-de07-4079-a7b2-679ee2c84172 kind: Event lastTimestamp: "2026-09-17T18:12:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:56Z" name: success-200-isvc-e7d45-predictor.18d62e3e22c8ac83 namespace: kserve-ci-e2e-test resourceVersion: "26797" uid: 6375ed88-50a6-4986-9b68-9b751f083a0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 38775a2e-de07-4079-a7b2-679ee2c84172 kind: Event lastTimestamp: "2026-09-17T18:13:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:56Z" name: success-200-isvc-e7d45-predictor.18d62e489f2182bb namespace: kserve-ci-e2e-test resourceVersion: "27309" uid: 50f17712-864f-4a3f-b532-cdb728bc4ddf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:13:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d45-predictor namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 38775a2e-de07-4079-a7b2-679ee2c84172 kind: Event lastTimestamp: "2026-09-17T18:13:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:56Z" name: success-200-isvc-e7d45-predictor.18d62e489fab47de namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: 242f0344-3cb6-4732-9499-d150bad35f65 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:12:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 927a7eff-4b69-4c10-a285-1f6e8d4e15c8 kind: Event lastTimestamp: "2026-09-17T18:12:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:59Z" name: success-200-isvc-e7d45.18d62e3aa3eec0b4 namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: afc8c621-6266-4b04-b555-6e634b0a8544 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7d45 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: 927a7eff-4b69-4c10-a285-1f6e8d4e15c8 kind: Event lastTimestamp: "2026-09-17T18:12:59Z" message: InferenceService [success-200-isvc-e7d45] is Ready metadata: creationTimestamp: "2026-09-17T18:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:59Z" name: success-200-isvc-e7d45.18d62e45e29e8025 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 461dab7c-32dd-4f00-beed-cda16a3f40e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:52:20.804759Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8592d-688fc594f9-lcxv9 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:52:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:52:20Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d2573b717a9 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: a13279e2-02ef-4e32-9263-0c2de696f1bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T17:52:21Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:52:21Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d259609eca8 namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 159c930c-7e9e-4bf1-b18a-f6d433f14cee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8592d} kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T17:52:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T17:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:21Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d259838c214 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 1149f276-654b-418d-a8de-70a7eef79ac0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8592d} kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T17:52:21Z" message: 'Created container: switch-graph-8592d' metadata: creationTimestamp: "2026-09-17T17:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:21Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d259b408557 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: c92ea92a-8760-47cf-bc59-f11c61e89c00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8592d} kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T17:52:21Z" message: Started container switch-graph-8592d metadata: creationTimestamp: "2026-09-17T17:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:52:21Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d259bc5b90b namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 562bb284-6983-4746-a6dc-ab2998d2c734 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8592d} kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T18:00:35Z" message: Stopping container switch-graph-8592d metadata: creationTimestamp: "2026-09-17T18:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:00:35Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d98a6e6ea4e namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: b3771e14-c8bf-4b04-b231-67cc29510719 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8592d} kind: Pod name: switch-graph-8592d-688fc594f9-lcxv9 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 01585e3c-a3a1-4b0d-b15d-52eedb9ccaa9 kind: Event lastTimestamp: "2026-09-17T18:01:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:01:03Z" name: switch-graph-8592d-688fc594f9-lcxv9.18d62d994988a440 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 2c7d2ef1-2bd6-4fca-a371-840cbcee8f33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8592d-688fc594f9 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 6dab4070-6d11-4c59-9e85-9ce103c75599 kind: Event lastTimestamp: "2026-09-17T17:52:20Z" message: 'Created pod: switch-graph-8592d-688fc594f9-lcxv9' metadata: creationTimestamp: "2026-09-17T17:52:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:20Z" name: switch-graph-8592d-688fc594f9.18d62d2573071a55 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: a750410c-f8c8-4ddb-b530-f130b7a9c2a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8592d namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: b41455a1-afb5-4beb-86bb-3173aa7a37b1 kind: Event lastTimestamp: "2026-09-17T17:52:20Z" message: Scaled up replica set switch-graph-8592d-688fc594f9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:52:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:20Z" name: switch-graph-8592d.18d62d25723a09c9 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 0e47e003-5541-4e93-b60a-6f2bdbcb7824 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:52:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8592d namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 069634e5-5ee9-4832-a0e2-96653f5dade2 kind: Event lastTimestamp: "2026-09-17T17:52:20Z" message: InferenceGraph [switch-graph-8592d] is Ready metadata: creationTimestamp: "2026-09-17T17:52:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:52:20Z" name: switch-graph-8592d.18d62d2578261142 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: bfb15edc-938e-473e-a987-635eb27cb030 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8592d namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 17df64ac-bc4b-4d7f-a16b-a2ddade85900 kind: Event lastTimestamp: "2026-09-17T17:52:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:50Z" name: switch-graph-8592d.18d62d28f090b585 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 3ea3f84e-1395-4e2b-a8bc-af8e28cdd73a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:52:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8592d namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 17df64ac-bc4b-4d7f-a16b-a2ddade85900 kind: Event lastTimestamp: "2026-09-17T17:52:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:52:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:52:50Z" name: switch-graph-8592d.18d62d28f0ff9af5 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 917a1c67-95c7-4bac-ac8e-92c61e6077ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:12:51.194888Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-85bfa-7886b55cbc-fnw2h to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-09-17T18:12:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:12:51Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e43ecae7d62 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: bb0857f1-8c2a-4997-a113-e88568547a76 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:12:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-85bfa-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:51Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e43f534ae3a namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 2c13153b-6bec-46fb-abfd-df003d0150ea reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:12:52Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:12:52Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e4432783b6f namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 3ff388bf-a17e-47bb-b45c-7bd40eefb552 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85bfa} kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:12:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" already present on machine metadata: creationTimestamp: "2026-09-17T18:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:52Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e4433d48453 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 902d478e-6fb1-49e0-82b0-6cd25880d90f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85bfa} kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:12:52Z" message: 'Created container: switch-graph-85bfa' metadata: creationTimestamp: "2026-09-17T18:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:52Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e4436bf33be namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: d7d5b0ea-d931-4efe-a06b-0a53d09505bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85bfa} kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:12:52Z" message: Started container switch-graph-85bfa metadata: creationTimestamp: "2026-09-17T18:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:12:52Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62e44372bc1a5 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: d77cef63-978e-44a1-9df3-d75817a98c23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85bfa} kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:29:10Z" message: Stopping container switch-graph-85bfa metadata: creationTimestamp: "2026-09-17T18:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:10Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62f27e00d4810 namespace: kserve-ci-e2e-test resourceVersion: "33611" uid: c049df89-d966-4476-aca8-0d6cfa215cfd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:29:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85bfa} kind: Pod name: switch-graph-85bfa-7886b55cbc-fnw2h namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6d83c913-069b-43b9-af7a-b332934a9840 kind: Event lastTimestamp: "2026-09-17T18:29:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:29:39Z" name: switch-graph-85bfa-7886b55cbc-fnw2h.18d62f28d4286cda namespace: kserve-ci-e2e-test resourceVersion: "33895" uid: 93075e43-9ae1-4800-8efe-27571af3fe34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-85bfa-7886b55cbc namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: f221d2c9-2d08-401c-ad74-6819e88be788 kind: Event lastTimestamp: "2026-09-17T18:12:51Z" message: 'Created pod: switch-graph-85bfa-7886b55cbc-fnw2h' metadata: creationTimestamp: "2026-09-17T18:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:51Z" name: switch-graph-85bfa-7886b55cbc.18d62e43ec47d13b namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: 97240e96-df2d-4f5e-b4f2-6efe840caa84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: 42fbd6d7-6feb-44ac-9b49-5d439cd047fc kind: Event lastTimestamp: "2026-09-17T18:12:51Z" message: Scaled up replica set switch-graph-85bfa-7886b55cbc from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:12:51Z" name: switch-graph-85bfa.18d62e43eb6ecfe5 namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: cac816d9-26aa-4f24-be1f-250c013627ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:12:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 342a2694-a62b-4e98-9e31-16a80a90b571 kind: Event lastTimestamp: "2026-09-17T18:12:51Z" message: InferenceGraph [switch-graph-85bfa] is Ready metadata: creationTimestamp: "2026-09-17T18:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:12:51Z" name: switch-graph-85bfa.18d62e43f0dc1bb1 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 1c00256f-7367-4c01-b674-28306954b4e6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: ba2bc0e4-c993-4c0e-8fa8-6cda7e89c05c kind: Event lastTimestamp: "2026-09-17T18:13:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:06Z" name: switch-graph-85bfa.18d62e476a1a8692 namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: e571c5b6-baac-457d-9dff-685168fa92ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: ba2bc0e4-c993-4c0e-8fa8-6cda7e89c05c kind: Event lastTimestamp: "2026-09-17T18:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:06Z" name: switch-graph-85bfa.18d62e476a6839e0 namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: 84af9f86-63b3-49d0-a25e-3bf6f2053977 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: ba2bc0e4-c993-4c0e-8fa8-6cda7e89c05c kind: Event lastTimestamp: "2026-09-17T18:13:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:21Z" name: switch-graph-85bfa.18d62e4ae96f840b namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: ff625a1f-1468-4d28-ad85-66aa2934a7fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-85bfa namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: ba2bc0e4-c993-4c0e-8fa8-6cda7e89c05c kind: Event lastTimestamp: "2026-09-17T18:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:13:21Z" name: switch-graph-85bfa.18d62e4ae9b59d20 namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: a8544d4f-3a77-4d26-bda5-84d00ea41949 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:51:10.355707Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d46ec-7f8c97c56d-rqgm4 to ip-10-0-135-40.ec2.internal metadata: creationTimestamp: "2026-09-17T17:51:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:51:10Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d150c9fd523 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: c5921560-bc11-468a-9409-09726da6bf8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6fd5d777c4-bmvp7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d46ec-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:10Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d1515c7a6b3 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: d866ae1e-2e95-4a6c-ac68-a080fa5017c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:11Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:51:11Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d155229f5f5 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: edc1a4d0-0ca3-4675-925a-107a0965900b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" metadata: creationTimestamp: "2026-09-17T17:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:11Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d155394a3f2 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: ed5558b6-d572-4496-812c-9f16110af06c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:bb50e7052755d34993c754ebb5475c57cbe2445f423ca013a14a621d1f8fda77" in 2.004s (2.004s including waiting). Image size: 211751507 bytes.' metadata: creationTimestamp: "2026-09-17T17:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:13Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d15cb15f34b namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 63a20b15-cf79-431c-8a25-24c8e39be756 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:13Z" message: 'Created container: switch-graph-d46ec' metadata: creationTimestamp: "2026-09-17T17:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:13Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d15ce4c5a2f namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 8eeb44ef-6086-445b-8ca3-f2c026423a21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:13Z" message: Started container switch-graph-d46ec metadata: creationTimestamp: "2026-09-17T17:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:13Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d15cebe8644 namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 2ebb7d0b-5130-4b04-aa09-69abe92159ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:20Z" message: Stopping container switch-graph-d46ec metadata: creationTimestamp: "2026-09-17T17:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:20Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d176cae75c3 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 10dabdfc-df8f-4c6c-aed5-07b35aad07eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d46ec} kind: Pod name: switch-graph-d46ec-7f8c97c56d-rqgm4 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 09dca777-3158-4cbd-b43c-f75074395091 kind: Event lastTimestamp: "2026-09-17T17:51:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:51:50Z" name: switch-graph-d46ec-7f8c97c56d-rqgm4.18d62d1875c2a20c namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 4515f174-8c1d-4df6-b17a-df7f697fd231 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-40.ec2.internal source: component: kubelet host: ip-10-0-135-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d46ec-7f8c97c56d namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: f107505a-6be1-45a8-b362-251030b79f31 kind: Event lastTimestamp: "2026-09-17T17:51:10Z" message: 'Created pod: switch-graph-d46ec-7f8c97c56d-rqgm4' metadata: creationTimestamp: "2026-09-17T17:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:10Z" name: switch-graph-d46ec-7f8c97c56d.18d62d150bf61c5c namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: bb1d4b8e-f7c2-4ab3-94df-a64fb5761bfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d46ec namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: a9038856-5399-4738-a9ab-c49405c645c6 kind: Event lastTimestamp: "2026-09-17T17:51:10Z" message: Scaled up replica set switch-graph-d46ec-7f8c97c56d from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:51:10Z" name: switch-graph-d46ec.18d62d150b172246 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 774939eb-f944-4a55-af1d-530ca2437f68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:51:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d46ec namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 9b3aa36f-9d30-4f28-99f0-775ff56900e9 kind: Event lastTimestamp: "2026-09-17T17:51:10Z" message: InferenceGraph [switch-graph-d46ec] is Ready metadata: creationTimestamp: "2026-09-17T17:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:51:10Z" name: switch-graph-d46ec.18d62d151148d889 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 3ed001db-dd4b-4a6f-9427-563ad7a938f9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33897"