--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:56:04.407961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-2b6c1-587666d7dc-k2xzd to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:56:04Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d5983812dad namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: ebc35f27-9e13-417a-9054-4996e58e4385 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:56:04Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d59a5fc36d7 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 684ddd4a-3443-4ac2-8cd8-0e866e406008 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2b6c1} kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:05Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d59a7478864 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: e7390193-6ea2-483c-9523-d72f586bd55b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2b6c1} kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: 'Created container: ensemble-graph-2b6c1' metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:05Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d59aab7f04f namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 2983501b-e7d1-4f9b-ae07-ed46e8b89e73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2b6c1} kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: Started container ensemble-graph-2b6c1 metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:05Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d59ab29ca22 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 5123632f-411f-425b-9ac5-3a09879ab834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2b6c1} kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Stopping container ensemble-graph-2b6c1 metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d5cc9b198e7 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: df0e8284-7427-4910-a724-c489546d065c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2b6c1} kind: Pod name: ensemble-graph-2b6c1-587666d7dc-k2xzd namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 30f336a7-8972-495d-87df-886d7f5b74b4 kind: Event lastTimestamp: "2026-09-17T17:56:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:46Z" name: ensemble-graph-2b6c1-587666d7dc-k2xzd.18d62d5d81d8b496 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: acebf8ea-6f32-41e8-84c4-9e8231a0b27d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-2b6c1-587666d7dc namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 1e0d9440-eb41-4d2f-8f85-c88236e33d7e kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'Created pod: ensemble-graph-2b6c1-587666d7dc-k2xzd' metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: ensemble-graph-2b6c1-587666d7dc.18d62d5983153ff1 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: bffe6020-0cdd-4d21-9dfa-a285d4c64bed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 6726ee47-593f-4f28-b32f-25fb4044f5df kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Scaled up replica set ensemble-graph-2b6c1-587666d7dc from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: ensemble-graph-2b6c1.18d62d598229a9bc namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 9a0fbdda-2589-4455-b3a6-a1e1740deedc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: f4a98e4f-85a2-4dc8-af76-3853326dd128 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: InferenceGraph [ensemble-graph-2b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:56:04Z" name: ensemble-graph-2b6c1.18d62d5987edeec6 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: ec069b4e-0752-4307-9e3a-3a6ab93539bc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:18.728324Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c5c33-68c85bbb54-slzk5 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:18Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6ad157f116 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: d337b888-8c3d-407c-b2cc-e483698fe62e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T17:57:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-c5c33-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:18Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6add9d894a namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: fee5edd2-35fa-4233-9e20-8a57d2f275f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T17:57:19Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:19Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6b1722aa74 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: f4eb030b-ecc1-4009-9a0b-087840c91747 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c5c33} kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T17:57:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:19Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6b18775c5b namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 1f4b1467-6c24-4bf0-a081-088ebdd0d989 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c5c33} kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T17:57:19Z" message: 'Created container: ensemble-graph-c5c33' metadata: creationTimestamp: "2026-09-17T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:19Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6b1beb2947 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: e7fbea4d-a99a-4a26-94df-1bab340a3fa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c5c33} kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T17:57:19Z" message: Started container ensemble-graph-c5c33 metadata: creationTimestamp: "2026-09-17T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:19Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62d6b1c57b99a namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 4df852fb-612e-492b-a481-cde72b9809b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c5c33} kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Stopping container ensemble-graph-c5c33 metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:33Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62dddf9797895 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 829f9305-9018-496d-b319-26b63b274b1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c5c33} kind: Pod name: ensemble-graph-c5c33-68c85bbb54-slzk5 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: a5dae2f9-d6ee-4d47-9c29-43d447468fed kind: Event lastTimestamp: "2026-09-17T18:06:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:01Z" name: ensemble-graph-c5c33-68c85bbb54-slzk5.18d62ddecd8fe58a namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: d87bfbed-ae6f-4fd4-8c40-e2f3c11bd8b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c5c33-68c85bbb54 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: d5c81d10-c8ca-49d7-a580-a21b419a1435 kind: Event lastTimestamp: "2026-09-17T17:57:18Z" message: 'Created pod: ensemble-graph-c5c33-68c85bbb54-slzk5' metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:18Z" name: ensemble-graph-c5c33-68c85bbb54.18d62d6ad0f6a673 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: c339cecf-71c8-4966-8cb9-d895029ae1fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: edca6153-d0fc-4c39-ad13-61fb5d3d9496 kind: Event lastTimestamp: "2026-09-17T17:57:18Z" message: Scaled up replica set ensemble-graph-c5c33-68c85bbb54 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:18Z" name: ensemble-graph-c5c33.18d62d6acff1ef73 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 1bdf4d0e-25af-4947-b75a-2fffceee968d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 0d9a4a37-1542-4f94-b325-1853fb60fcb4 kind: Event lastTimestamp: "2026-09-17T17:57:18Z" message: InferenceGraph [ensemble-graph-c5c33] is Ready metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:18Z" name: ensemble-graph-c5c33.18d62d6ad517f81b namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 30f57d0a-8740-4aef-b4be-00025c068983 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 47c8bf68-b66b-4694-9e6c-634782494e5f kind: Event lastTimestamp: "2026-09-17T17:57:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:48Z" name: ensemble-graph-c5c33.18d62d6e4ef1f332 namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 8dd28bf3-266a-4dc0-b5ea-e853b62bdeae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 47c8bf68-b66b-4694-9e6c-634782494e5f kind: Event lastTimestamp: "2026-09-17T17:57:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:48Z" name: ensemble-graph-c5c33.18d62d6e4f6bd0d7 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 410d641d-6569-4bae-894a-9d1f17dce7ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:55:04.465544Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4b8ea86bee namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: b3ddc19d-1ce3-4706-95d0-6cc004fd2e70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2b6c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4ba0ada327 namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: 8407f84d-8638-42d5-a28c-09900f1791ff reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bd46f54bc namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: e2d24044-4246-48b3-ad3b-ff11750f7d12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bd5d53d84 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: a6311ab1-53bd-4f5b-ad05-2ed6fb5a9f54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bd99a28de namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 7b294678-bcfb-4bcf-9109-e5ad0ba03556 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bda103658 namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: d1a6f3ba-51f6-49d0-a7ec-f5cb879ae6d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bda1b042d namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 386210ab-02e8-4af2-b967-3b2fde9dc89e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bdcd17544 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 74599709-e0e1-44bc-87bd-f5ed2ce1b8f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4bdd464975 namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: cd171ace-73e4-4899-88f7-a56f2994995d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:55:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:43Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d4c4f585616 namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: 05181595-cdc1-4187-9e3a-ce5697dc570a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d5cd3216b07 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: db8ca424-a1a4-4b34-91c0-79c835da1f9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d5cd32208d9 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 627acb82-d082-4145-9525-163c58dbd18a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: a598baa1-44e2-4711-9d08-a245a368b3eb kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf.18d62d5cd7d3f1d4 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 24b2e99c-8e03-42f7-afe4-ea5475a8d061 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2b6c1-predictor-88cbb7679 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: c84cb07d-1122-4cc4-ab00-f781777022c7 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'Created pod: error-404-isvc-2b6c1-predictor-88cbb7679-pf5xf' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1-predictor-88cbb7679.18d62d4b8e2c63ee namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 4ead6a60-135d-4e59-b1a8-ddfddeee7074 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 376acd32-906f-4827-9d1d-ecfe14e8e672 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Scaled up replica set error-404-isvc-2b6c1-predictor-88cbb7679 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1-predictor.18d62d4b8c766e9d namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 0d8fab89-f3df-4626-b3e9-937c903b1f56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 25e52ac4-3b0f-4a12-b5f6-b2ad92d54ca0 kind: Event lastTimestamp: "2026-09-17T17:55:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:34Z" name: error-404-isvc-2b6c1-predictor.18d62d4f0ae2e1f1 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 7fef7cc2-6b04-491b-aa6b-20a2203b6ef7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 25e52ac4-3b0f-4a12-b5f6-b2ad92d54ca0 kind: Event lastTimestamp: "2026-09-17T17:55:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:34Z" name: error-404-isvc-2b6c1-predictor.18d62d4f0b4725c7 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: e5304731-a353-4e3c-9b06-8209a0ab8ccd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 25e52ac4-3b0f-4a12-b5f6-b2ad92d54ca0 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: error-404-isvc-2b6c1-predictor.18d62d5608bca3c9 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: f34554e5-1c23-4aa9-b0ab-85c41b903468 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 25e52ac4-3b0f-4a12-b5f6-b2ad92d54ca0 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: error-404-isvc-2b6c1-predictor.18d62d560925d878 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 89d1d24a-06f0-4485-b582-9765bbeccb55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 77879ec6-4abb-4ebb-85c3-b781cbfb5653 kind: Event lastTimestamp: "2026-09-17T17:55:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:53Z" name: error-404-isvc-2b6c1.18d62d4b8ca0e0bc namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 46ebb449-5878-4597-9bbc-285293a2b804 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 77879ec6-4abb-4ebb-85c3-b781cbfb5653 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-2b6c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1.18d62d4b8d08d43d namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 8ccd81d0-2e49-46f0-b030-04bc5aa7570b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 77879ec6-4abb-4ebb-85c3-b781cbfb5653 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2b6c1.18d62d4b8d0915eb namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 3b777aec-1eb5-4a3d-a2cb-87e437cb9f77 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: 77879ec6-4abb-4ebb-85c3-b781cbfb5653 kind: Event lastTimestamp: "2026-09-17T17:55:53Z" message: InferenceService [error-404-isvc-2b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:55:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:53Z" name: error-404-isvc-2b6c1.18d62d570b5a77a8 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 71518dcd-9a21-4c3f-a312-bbcd6a597679 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:45:49.905321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2d039-predictor-c5df97d75-lhdfb to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:45:49Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca704b14d5 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 6481c11d-2058-441f-97cf-d522e96dfe7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca92750ef4 namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: aec1400a-1af8-49ed-85f8-333117aa8e79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca93f1afe3 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: d22d10ec-e0e3-4bc8-ba31-97058bd19450 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca96b7a4f7 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: c1b1a153-ea7f-489f-8752-e96317fba4c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca97464922 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 9db966c3-b963-4237-ba71-d57cb5df2a27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca975079a1 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 19c5c37b-e526-4b18-bb30-a8b0ff082d9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca9b2b7a1c namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 3676f1cc-d925-49a1-b087-1bbfeac79f92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62cca9b98f3d5 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: b1138130-9194-402c-9970-f41b49e3206f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:46:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:28Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62ccaf6c6a539 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: f4628f48-17e3-4cf8-9970-1b5d4c90c706 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62d4b8781ef51 namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 605a999a-25be-4e2b-a6aa-64b233bccf45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2b144984-69c6-4be8-abfb-d9533de0a0e5 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: error-404-isvc-2d039-predictor-c5df97d75-lhdfb.18d62d4b878325e3 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 1bc927a3-a325-4d2f-9368-ae492b09bba0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2d039-predictor-c5df97d75 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 3bc210a0-a73b-4ed6-ae44-1ea1af431724 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: 'Created pod: error-404-isvc-2d039-predictor-c5df97d75-lhdfb' metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:49Z" name: error-404-isvc-2d039-predictor-c5df97d75.18d62cca6f392956 namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 1a473d64-6724-497f-b414-0ac13505641d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 41d09ebf-010e-4a16-ae65-da366e35fe4b kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Scaled up replica set error-404-isvc-2d039-predictor-c5df97d75 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:49Z" name: error-404-isvc-2d039-predictor.18d62cca6aa5c431 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 227d6638-cc37-4b63-b85d-ddd0bdcd2bc5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 645bb714-0fcd-49ff-8990-586ec0f5a6da kind: Event lastTimestamp: "2026-09-17T17:46:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:19Z" name: error-404-isvc-2d039-predictor.18d62ccde9197269 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 4ca6053c-d5f9-48c3-b288-e96ad5b1bfab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 645bb714-0fcd-49ff-8990-586ec0f5a6da kind: Event lastTimestamp: "2026-09-17T17:46:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:19Z" name: error-404-isvc-2d039-predictor.18d62ccde9888bd8 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: f4831a34-5887-4756-ba2b-e7ad9e362f3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 645bb714-0fcd-49ff-8990-586ec0f5a6da kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:19Z" name: error-404-isvc-2d039-predictor.18d62cd4e6fa404b namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 8506f72b-90f0-4414-a906-2f49239ca6fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 645bb714-0fcd-49ff-8990-586ec0f5a6da kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:19Z" name: error-404-isvc-2d039-predictor.18d62cd4e7582178 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 87233839-caf0-4b52-910c-c8417cc90325 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 9e1ab3ae-ec50-426b-9560-a3f691de7c5e kind: Event lastTimestamp: "2026-09-17T17:46:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:38Z" name: error-404-isvc-2d039.18d62cca6ac7e51d namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: ee4dec7b-cf48-4094-b1f4-1ef4ea558151 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9e1ab3ae-ec50-426b-9560-a3f691de7c5e kind: Event lastTimestamp: "2026-09-17T17:46:38Z" message: InferenceService [error-404-isvc-2d039] is Ready metadata: creationTimestamp: "2026-09-17T17:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:38Z" name: error-404-isvc-2d039.18d62cd5b5bf8f4a namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 1a994245-c30c-43e5-a99f-90eab67e6af6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:50.261722Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-390a8-predictor-56ccd8877-qp6qj to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:50Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd87dd0966b namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 2f19b9a8-f711-40e0-a33a-1e658031c620 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-390a8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd88dc0c6b3 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: ffb05028-f71a-4861-b15a-ab274466cc39 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd8f94a7e4b namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: a60a9ec4-95c5-47a0-a1d9-346c4964ccff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd8faaa98d1 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 7cb44e34-88bd-4e9a-ac47-1aab34961c11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd8fda8ec66 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 0878e602-a8bd-4f50-bc30-5e153a66f23a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd8fe1adb8a namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 2cc3b7d1-6a46-4fb4-8632-de7e914a4a26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd8fe23aed4 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 4cf18f6b-baaf-40a6-b99a-0ecb216d21a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd90267f5d3 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 411302ba-6f7a-4a06-99d7-d8c67264dcee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd902d8fe4c namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 2c813e57-5e5f-4ad0-a396-c228c21f45f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:47:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:29Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62cd93b99b334 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 46dfc597-60d5-4354-9b6d-c2368f58b9eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:04Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62d59a15ecd13 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 905fe8d4-5d2a-48f0-99b1-e717d3d11e94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: f6fd4091-5a8d-4971-b38f-7659800208d0 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:04Z" name: error-404-isvc-390a8-predictor-56ccd8877-qp6qj.18d62d59a15f18c7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: f9364343-6532-4b91-9ff5-4b04755f1a79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-390a8-predictor-56ccd8877 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 716dc5a6-91a7-4903-9240-5b65c9c36afc kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: 'Created pod: error-404-isvc-390a8-predictor-56ccd8877-qp6qj' metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:50Z" name: error-404-isvc-390a8-predictor-56ccd8877.18d62cd87d33f7bf namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: c071f6c3-8198-4fe4-9712-d483e0d3146d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 98d69354-5667-417f-a4fa-200d32500a35 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Scaled up replica set error-404-isvc-390a8-predictor-56ccd8877 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:50Z" name: error-404-isvc-390a8-predictor.18d62cd87b75f616 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 3faaca8a-bf09-42f4-805d-59725028abc8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 18111d8a-2b21-4be2-8a24-e50c34ea4cd5 kind: Event lastTimestamp: "2026-09-17T17:47:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:20Z" name: error-404-isvc-390a8-predictor.18d62cdbfa39792f namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 516a4884-37d5-4541-bef1-5e6df77a4e7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 18111d8a-2b21-4be2-8a24-e50c34ea4cd5 kind: Event lastTimestamp: "2026-09-17T17:47:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:20Z" name: error-404-isvc-390a8-predictor.18d62cdbfaae7c60 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: ff900261-1890-4760-bd23-5d667ec2d726 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 18111d8a-2b21-4be2-8a24-e50c34ea4cd5 kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:05Z" name: error-404-isvc-390a8-predictor.18d62ce2f8060633 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 9b51e4ae-d723-44ce-83be-50a6b17805b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 18111d8a-2b21-4be2-8a24-e50c34ea4cd5 kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:05Z" name: error-404-isvc-390a8-predictor.18d62ce2f8502311 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 87ebc022-81f3-46e8-9aeb-1867753dc0a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390a8 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: f2292a59-90ae-4dda-9bda-d50a6d3efb11 kind: Event lastTimestamp: "2026-09-17T17:47:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:39Z" name: error-404-isvc-390a8.18d62cd87b828cb0 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 5adafaa5-2f84-4fb9-83f7-643c5d89f3e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: f2292a59-90ae-4dda-9bda-d50a6d3efb11 kind: Event lastTimestamp: "2026-09-17T17:47:39Z" message: InferenceService [error-404-isvc-390a8] is Ready metadata: creationTimestamp: "2026-09-17T17:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:39Z" name: error-404-isvc-390a8.18d62ce3f77461d1 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 4cf94557-1597-4396-a9cf-859609b6ae89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:06:43.977811Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62dee6cd69497 namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: a540d637-484e-42f6-b368-d5322709e06f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3ef6b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62dee74bd99c8 namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 155d2864-df89-4ced-bf18-7cfc3c7da984 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Add eth0 [10.133.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deeb34165c8 namespace: kserve-ci-e2e-test resourceVersion: "25865" uid: c2ffabd7-8f7e-4c83-a8c4-16fde29e75a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deeb48745f3 namespace: kserve-ci-e2e-test resourceVersion: "25867" uid: d010430e-9842-467b-88e7-d115ec2d8df3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deeb8abd826 namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: e44d4e60-834b-491e-95c5-0359826930ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deeb9693b16 namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: a56a54d2-4643-4940-b1e6-0201ca2021bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deeb97a6157 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: 74415012-9acc-40cc-a5e6-2aabce11282f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deebd0047ec namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: fc74283c-8efc-4210-8e2f-6e609d2313b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deebd797b1c namespace: kserve-ci-e2e-test resourceVersion: "25873" uid: aaa6ea82-4302-4352-ba69-9fe70929523d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:07:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:22Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62deee45eb858 namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: 48849129-acb7-4165-8228-a2eaa82b288a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:15:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:15:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:15:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:15:58Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62e6f93c57d0a namespace: kserve-ci-e2e-test resourceVersion: "30326" uid: 9c5b8608-a253-4ef2-af25-7b47913cf414 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:15:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: aeb32779-c788-4434-8d88-5ec5d08e7144 kind: Event lastTimestamp: "2026-09-17T18:15:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:15:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:15:58Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9.18d62e6f93c5dae0 namespace: kserve-ci-e2e-test resourceVersion: "30327" uid: cbb1d508-f445-4901-b2e0-62bab6b9d9cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3ef6b-predictor-fcf5d54cf namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: bada0586-280a-4280-b367-2664fcfb82e7 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'Created pod: error-404-isvc-3ef6b-predictor-fcf5d54cf-rcnf9' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-3ef6b-predictor-fcf5d54cf.18d62dee6c4b76c2 namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 072d88dd-fb32-442a-9693-e9ad3ab85056 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 94df5233-1a5b-445a-aa87-89ee1c552d51 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Scaled up replica set error-404-isvc-3ef6b-predictor-fcf5d54cf from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-3ef6b-predictor.18d62dee6a5369f5 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 0a176e83-9dfc-471c-be9a-1652236b1be7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 006009cc-230d-4176-9571-46ed48fcfa2d kind: Event lastTimestamp: "2026-09-17T18:07:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:13Z" name: error-404-isvc-3ef6b-predictor.18d62df1e8f7350d namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: b740f13b-895a-4f68-818f-cde860e1422d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 006009cc-230d-4176-9571-46ed48fcfa2d kind: Event lastTimestamp: "2026-09-17T18:07:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:13Z" name: error-404-isvc-3ef6b-predictor.18d62df1e93d8bfe namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 915dbb22-6c34-4341-9ac7-93a1b5a77da8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 006009cc-230d-4176-9571-46ed48fcfa2d kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: error-404-isvc-3ef6b-predictor.18d62df8e735dae8 namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: 4e22955e-5faa-4b2f-907a-a1f0a956e1f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 006009cc-230d-4176-9571-46ed48fcfa2d kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: error-404-isvc-3ef6b-predictor.18d62df8e7d79b96 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 6cdbb195-d494-42d4-8254-6556217c7edb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: cc703b5e-c170-4e65-8dc8-80cdd0811c02 kind: Event lastTimestamp: "2026-09-17T18:07:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:32Z" name: error-404-isvc-3ef6b.18d62dee6a997c4c namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 0ddf762d-1f10-420e-8299-3f7beeb63030 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: cc703b5e-c170-4e65-8dc8-80cdd0811c02 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-3ef6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ef6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-3ef6b.18d62dee6b1b522a namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 41ba8067-1c48-4f56-84a9-d5a854b0b115 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: cc703b5e-c170-4e65-8dc8-80cdd0811c02 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ef6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-3ef6b.18d62dee6b1b8ce7 namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 3ed21c72-ba43-4693-acc4-ce6dc2f1b38e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: cc703b5e-c170-4e65-8dc8-80cdd0811c02 kind: Event lastTimestamp: "2026-09-17T18:07:32Z" message: InferenceService [error-404-isvc-3ef6b] is Ready metadata: creationTimestamp: "2026-09-17T18:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:32Z" name: error-404-isvc-3ef6b.18d62df9a480336b namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 089e8924-a4ee-4232-8d4e-fb6efcaf343e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:44:19.776836Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5b8c1-predictor-6d6d86965-4rplj to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:44:19Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb5743790bb namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 803842bf-ecff-404e-becc-4d8898ae22f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5b8c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb581d26f40 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 2a630627-6338-412d-bc19-c8548740d532 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:21Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:44:21Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb5f2501d54 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 8d506f03-6979-431e-ad17-a54b2f33233f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:21Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" metadata: creationTimestamp: "2026-09-17T17:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:21Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb5f41662f4 namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: 0f5a077f-0a5f-4308-aa32-b3752f932b04 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" in 15.764s (15.764s including waiting). Image size: 1355794496 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb99fc0d3ca namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 3a8e166b-54d2-4dbc-b782-0137702fa58b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb9a2c2aac0 namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: abdbaa22-d3d6-4c72-b34d-656c09f08945 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb9a327a9dd namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 92f79894-8cd4-4acc-898e-a5d9c00027b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cb9a335c5cd namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: bcb7611c-c016-4589-820c-72785ad060f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.67s (2.67s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cba42633a80 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: c9b4b844-8bb8-40ff-96ca-e38ba6cb24ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cba45c2570c namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 7174fb3b-b247-46e9-af98-c642a872dbd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cba46347d38 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 6ac51aeb-efe4-4e3d-9006-6647b5895144 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:45:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:16Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cba5c278ae7 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 5fc20df9-9308-40d2-a06a-2a420c18f935 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:49Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cca5df54e61 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 3922cfda-4c35-40a7-ad71-7572f77d0cac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:49Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62cca5df5cd4b namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: bed7a45e-8a51-427f-b92b-02bd6e1f7264 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 0d0e8cef-9085-4468-9430-9585789557b9 kind: Event lastTimestamp: "2026-09-17T17:45:51Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:51Z" name: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj.18d62ccae47cdcdd namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: c05df37c-6464-4859-9c10-ca54e468eb08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5b8c1-predictor-6d6d86965 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 2492ded5-a646-4533-b8fb-04bb1d824b0b kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'Created pod: error-404-isvc-5b8c1-predictor-6d6d86965-4rplj' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: error-404-isvc-5b8c1-predictor-6d6d86965.18d62cb5738bda0c namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: c36d225f-66b8-4858-96f9-e1699a759306 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 80c8af91-6ea6-45dd-bfa3-fc87be9332c2 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: Scaled up replica set error-404-isvc-5b8c1-predictor-6d6d86965 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: error-404-isvc-5b8c1-predictor.18d62cb571160285 namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: 627504fe-1457-4f29-8909-db2ec588d751 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: ca1dcf97-48ae-4d4a-a160-be15e1d35d1c kind: Event lastTimestamp: "2026-09-17T17:45:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:19Z" name: error-404-isvc-5b8c1-predictor.18d62cb8ef907b18 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 3701a48a-9595-4763-85d9-8e926de0fcd8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: ca1dcf97-48ae-4d4a-a160-be15e1d35d1c kind: Event lastTimestamp: "2026-09-17T17:45:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:19Z" name: error-404-isvc-5b8c1-predictor.18d62cb8f00c1b93 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 02d2fcc8-d727-4824-8da6-c789ac7828b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: ca1dcf97-48ae-4d4a-a160-be15e1d35d1c kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: error-404-isvc-5b8c1-predictor.18d62cbfed8674fe namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: b20bba76-d212-46cc-a84c-0b0d162bc0d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: ca1dcf97-48ae-4d4a-a160-be15e1d35d1c kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: error-404-isvc-5b8c1-predictor.18d62cbfedf72208 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: f893fa85-2015-45a5-8c33-23c2be48424a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: f3bdff0f-20ca-4121-b41f-c8fe40d81281 kind: Event lastTimestamp: "2026-09-17T17:45:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:45:26Z" name: error-404-isvc-5b8c1.18d62cb571394474 namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: e6ecb511-745b-477d-a690-76ca274fd9cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: f3bdff0f-20ca-4121-b41f-c8fe40d81281 kind: Event lastTimestamp: "2026-09-17T17:45:26Z" message: InferenceService [error-404-isvc-5b8c1] is Ready metadata: creationTimestamp: "2026-09-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:45:26Z" name: error-404-isvc-5b8c1.18d62cc517d31685 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: d5f50103-4966-4a9f-8db3-b35bd1254ceb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:56:05.106205Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6b6c1-predictor-66c99c6495-hh86l to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:56:05Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59ad1f8874 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: d2d2dc18-8cb9-4f67-9300-c71e48f437ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6b6c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:05Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59b653829d namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 16faaff1-e9b5-471c-a255-6dd3640201b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59f2ddbf39 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 8e1b2811-ff9d-461e-80bf-6b42b1a42782 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59f43a107e namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: f5aae705-32eb-485e-be6f-75c292319fd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59f6c96caf namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: f66f926f-773a-4a08-b523-f284f1b59745 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59f74e95b3 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: f7a291c1-88ff-448f-af75-f2e12dbc4aba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59f75bc0de namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b92f9d5a-9a10-407c-b3b0-908ee7c963de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59fa9491c9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: d0f6ad7c-e654-4b58-92a6-818af02d6a70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d59fb04db6b namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 0bf894eb-cae2-4caa-9c06-bd5efea7dc0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:56:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:43Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d5a1cc2ac64 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 106b6f47-cbd7-42bd-a498-dfa5d8800243 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d6d44227b3e namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 6f5cf14d-4b89-445b-8149-da7fc8005a53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: bd8084fe-ed00-4a05-a0a3-0ab206ba7278 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l.18d62d6d4423b675 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: e5f3b704-ebc4-4e68-87c8-c3c2829f439b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6b6c1-predictor-66c99c6495 namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 6777cef9-efa6-4478-a560-1a30e2cc96dd kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: 'Created pod: error-404-isvc-6b6c1-predictor-66c99c6495-hh86l' metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:05Z" name: error-404-isvc-6b6c1-predictor-66c99c6495.18d62d59acb2a524 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 26ae553f-5340-4c6a-995c-8c822a4e46e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 97ec7a27-aeb3-4c87-831b-ec72fa748961 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: Scaled up replica set error-404-isvc-6b6c1-predictor-66c99c6495 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:05Z" name: error-404-isvc-6b6c1-predictor.18d62d59aaf7d97f namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 1c377e80-f3c5-48c9-9a1c-6c26fd273c71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 8a1a8192-8374-428e-95e8-47b71211e464 kind: Event lastTimestamp: "2026-09-17T17:56:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:20Z" name: error-404-isvc-6b6c1-predictor.18d62d5d297c98d7 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 7f9d7682-2ced-4739-a1cf-8ca91c4d850e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 8a1a8192-8374-428e-95e8-47b71211e464 kind: Event lastTimestamp: "2026-09-17T17:56:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:20Z" name: error-404-isvc-6b6c1-predictor.18d62d5d29e66899 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: c9781d70-f122-4a11-bdcf-67f7b6f4da5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:56:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 8a1a8192-8374-428e-95e8-47b71211e464 kind: Event lastTimestamp: "2026-09-17T17:57:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:20Z" name: error-404-isvc-6b6c1-predictor.18d62d60a885fcb2 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 4966aacd-a214-42e5-a66d-31e5246f2107 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:56:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 8a1a8192-8374-428e-95e8-47b71211e464 kind: Event lastTimestamp: "2026-09-17T17:57:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:20Z" name: error-404-isvc-6b6c1-predictor.18d62d60a8f22644 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: f38e0b4f-3e93-4b58-a366-54e2ce07a8aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: f5defa68-dee8-467f-87ca-a8ff23107687 kind: Event lastTimestamp: "2026-09-17T17:56:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:56:53Z" name: error-404-isvc-6b6c1.18d62d59ab006aea namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: a03e3487-06a7-46df-bcfb-f432a7839d49 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: f5defa68-dee8-467f-87ca-a8ff23107687 kind: Event lastTimestamp: "2026-09-17T17:56:53Z" message: InferenceService [error-404-isvc-6b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:56:53Z" name: error-404-isvc-6b6c1.18d62d64d925514d namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 6e0687f6-1104-45ea-bf49-dd3df6ada1aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:05:33.670351Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b5897-predictor-658695b98f-5vcmf to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:05:33Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde0e2fe8f3 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: 60813546-9c5e-4c68-a342-01429697c871 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde304bee66 namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 1c1f48fe-a4c5-4556-8a3f-871e67160bbe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde31f9e676 namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: d015c75c-fad1-49a3-89ab-c09fefbd232d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde340f3f56 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4481946d-c369-4de1-8721-64c137ff0a45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde347b15ad namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: fd257520-8f7f-4425-8d54-5fc74437e518 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde3484feb1 namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 4b835092-d430-4805-93b8-9f43696e80d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde37d915f5 namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: 9e536840-f702-47f0-937f-444fec98e0e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde384c160a namespace: kserve-ci-e2e-test resourceVersion: "25026" uid: 86be032c-8c62-49f4-b8ae-1cf5d732827c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:06:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:11Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dde82a72eb1 namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 41d663aa-1590-4d56-a4f8-974ffaccaa24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dee639ee9bd namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: 1ec1f125-5858-4270-9456-885f6790afe3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62dee639f0cbb namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: 6573227c-34c4-4c23-9a66-8739d4c48ab7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5897-predictor-658695b98f-5vcmf namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 772aca13-71ef-4f5a-b76b-82e7451ea20a kind: Event lastTimestamp: "2026-09-17T18:06:46Z" message: 'Readiness probe failed: Get "https://10.133.0.47:8643/healthz": dial tcp 10.133.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:46Z" name: error-404-isvc-b5897-predictor-658695b98f-5vcmf.18d62def0adcffd6 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 0f47a520-c4cd-4938-85b3-1c4b9e51b4cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b5897-predictor-658695b98f namespace: kserve-ci-e2e-test resourceVersion: "24978" uid: 504d041e-c8df-43fe-b8f2-328af8cf2591 kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: 'Created pod: error-404-isvc-b5897-predictor-658695b98f-5vcmf' metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:05:33Z" name: error-404-isvc-b5897-predictor-658695b98f.18d62dde0dbfa8be namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: 3b457557-1efe-4e75-892c-3f30053d185d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: 8592fe00-12d8-4a78-8463-d87d94d0e3f5 kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Scaled up replica set error-404-isvc-b5897-predictor-658695b98f from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:05:33Z" name: error-404-isvc-b5897-predictor.18d62dde0bd57c64 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b71476b1-c7de-445a-a7bf-1efd47578566 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: f46abd6f-ddf2-4925-bd5f-822ec3905e1e kind: Event lastTimestamp: "2026-09-17T18:06:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:03Z" name: error-404-isvc-b5897-predictor.18d62de18a534abb namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: 617f9d27-611b-49e6-93be-c05df4f78726 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: f46abd6f-ddf2-4925-bd5f-822ec3905e1e kind: Event lastTimestamp: "2026-09-17T18:06:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:03Z" name: error-404-isvc-b5897-predictor.18d62de18ac1c040 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 5f3dfedb-bb04-4356-af12-4fd2eeeb97e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: f46abd6f-ddf2-4925-bd5f-822ec3905e1e kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: error-404-isvc-b5897-predictor.18d62de8881cec9e namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 7ea7ea4a-8956-411e-bcb0-dcc95da795c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: f46abd6f-ddf2-4925-bd5f-822ec3905e1e kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: error-404-isvc-b5897-predictor.18d62de8888f79fc namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: d5e9eb95-814e-4dec-8bba-9bdea3743a8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5897 namespace: kserve-ci-e2e-test resourceVersion: "24960" uid: cbf39865-3fde-4d96-b0a0-49ce7f3bfe64 kind: Event lastTimestamp: "2026-09-17T18:06:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:21Z" name: error-404-isvc-b5897.18d62dde0be185d4 namespace: kserve-ci-e2e-test resourceVersion: "25418" uid: a6b83050-aeae-4eb0-a52f-f6e203b48314 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5897 namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: cbf39865-3fde-4d96-b0a0-49ce7f3bfe64 kind: Event lastTimestamp: "2026-09-17T18:06:21Z" message: InferenceService [error-404-isvc-b5897] is Ready metadata: creationTimestamp: "2026-09-17T18:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:21Z" name: error-404-isvc-b5897.18d62de93f31d029 namespace: kserve-ci-e2e-test resourceVersion: "25414" uid: 0b4db486-d21b-4565-86aa-3171e34ec184 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:56:18.809827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5cddec3bc0 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 0b65857f-6a5f-411e-b392-66c9a74a6323 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d008ca9ce namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 6033dae5-0363-4b32-a58a-6812640068a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d021aab5a namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: c9b652a1-b86a-45f2-b170-04ac1b6a3829 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d06429d47 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 58af436d-535d-4508-a89e-d61b95aef777 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d06ba31df namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: dbd5977c-62d3-4859-a634-7914f31c99a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d06c44b92 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: fa34fbc1-c2c8-497a-bb6d-867878dc2d31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d0a5fde39 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 795b7c34-0970-453d-a18f-bd366085cfa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d0ad5687f namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 81b4368c-5f2a-455a-b593-11b03ad93801 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T17:56:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:57Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62d5d64317269 namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: b5ae87eb-e19c-461f-94c9-fb68dba7569a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:33Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62dde0290ecf9 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: d925146c-c623-4219-b534-3afb691237d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: b4205277-b668-41c9-974a-730d76d07f5a kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:33Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5.18d62dde0291664e namespace: kserve-ci-e2e-test resourceVersion: "24945" uid: 761f872e-74dd-417f-9812-cae8edff4e1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c5c33-predictor-6cd4d7bc49 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 90f1c87f-da90-4cea-a07c-a84ebaae26dc kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: 'Created pod: error-404-isvc-c5c33-predictor-6cd4d7bc49-lmkh5' metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-c5c33-predictor-6cd4d7bc49.18d62d5cdd77aa0b namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 7406b365-1545-4a3f-9f97-6debfe218365 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 79291a85-0db0-45c0-a88c-31bcb361995f kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Scaled up replica set error-404-isvc-c5c33-predictor-6cd4d7bc49 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:18Z" name: error-404-isvc-c5c33-predictor.18d62d5cdbc17cb5 namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 2320e6d1-d6a9-4e1b-8558-3b2bc136a0e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:56:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 8398901a-bfa2-4f5d-9ab4-da3141dc3b0e kind: Event lastTimestamp: "2026-09-17T17:56:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:48Z" name: error-404-isvc-c5c33-predictor.18d62d605a446c33 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: a815e0bf-e57d-4db4-909f-740d6d114726 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:56:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 8398901a-bfa2-4f5d-9ab4-da3141dc3b0e kind: Event lastTimestamp: "2026-09-17T17:56:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:48Z" name: error-404-isvc-c5c33-predictor.18d62d605aa9e953 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: c6fe093f-7a0f-42e9-86cc-63795e127407 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 8398901a-bfa2-4f5d-9ab4-da3141dc3b0e kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:33Z" name: error-404-isvc-c5c33-predictor.18d62d67582239ba namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: b6aa5d46-d727-4e4e-add1-e4aad5af79cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 8398901a-bfa2-4f5d-9ab4-da3141dc3b0e kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:33Z" name: error-404-isvc-c5c33-predictor.18d62d67586ff97b namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 6794a310-61f7-4236-9164-e87c87b66b28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 6ecfe184-921d-42a3-b736-fd94b79f6018 kind: Event lastTimestamp: "2026-09-17T17:57:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:07Z" name: error-404-isvc-c5c33.18d62d5cdbd138c3 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 2b86553f-7a9d-46a9-aa07-21db815f4fda reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 6ecfe184-921d-42a3-b736-fd94b79f6018 kind: Event lastTimestamp: "2026-09-17T17:57:07Z" message: InferenceService [error-404-isvc-c5c33] is Ready metadata: creationTimestamp: "2026-09-17T17:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:07Z" name: error-404-isvc-c5c33.18d62d6823f0a48f namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: baf63f3c-314d-496c-95ef-8bf45b3f21c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:06:44.405535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25825" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-de145-predictor-bfc475677-phjm7 to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62dee8655243d namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 46a0947d-3daf-4fe4-86f1-bc1ad167a357 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-de145-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62dee92f6e639 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: 9d46304c-92cc-4bce-af1d-808c73341f85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25835" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Add eth0 [10.133.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deecca4f4b9 namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: f382ea33-f408-4a34-9a34-b1049f4b67b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deece23aae0 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 6a9107a5-37f1-45a6-94d8-7297bbd33c76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deed1c64274 namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: 33bee0a7-4e9b-46a6-8f38-7b7207b757fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deed2397172 namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 0fc87060-c0ed-4f22-a39e-8aa972b0dcba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deed244e643 namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: c79c6f18-1638-41a1-ad83-5b0c1dc8144c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deed54a3bba namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: c4200b4a-5458-4ad2-968c-ac75b725df83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62deed5b92c0b namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: dd7a657f-8551-41f6-8fa1-131405458b6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:07:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:23Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62def204f422e namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 30989fa0-e2f7-4121-be4d-f8720aebad7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:24:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:13Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62ee2c8624eb7 namespace: kserve-ci-e2e-test resourceVersion: "33765" uid: 74c1abb1-55fa-40ba-8165-b3e73ec9492e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de145-predictor-bfc475677-phjm7 namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 453eb767-5a5e-479c-8079-59038d055f71 kind: Event lastTimestamp: "2026-09-17T18:24:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:13Z" name: error-404-isvc-de145-predictor-bfc475677-phjm7.18d62ee2c862b255 namespace: kserve-ci-e2e-test resourceVersion: "33766" uid: 0f920fe2-9b44-4870-82bf-a4bfe39b1dbe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-de145-predictor-bfc475677 namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: f8c460f6-b505-4dbb-81d4-dbf3882e7e1a kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'Created pod: error-404-isvc-de145-predictor-bfc475677-phjm7' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-de145-predictor-bfc475677.18d62dee859e2147 namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: 5b5509e8-49bc-475f-99a2-ff3c4714e12b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 20ea0cf6-8bec-46f4-be24-d2521f2d00b6 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Scaled up replica set error-404-isvc-de145-predictor-bfc475677 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-de145-predictor.18d62dee83c0d82a namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: be35274b-a397-4aba-8f30-a67eac1d9976 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 09f3fb27-c3af-4eaa-9db8-86e1f11850f0 kind: Event lastTimestamp: "2026-09-17T18:07:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:14Z" name: error-404-isvc-de145-predictor.18d62df2024d2878 namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: e5f099c6-ba33-467a-bbce-52f0c3b90b6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 09f3fb27-c3af-4eaa-9db8-86e1f11850f0 kind: Event lastTimestamp: "2026-09-17T18:07:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:14Z" name: error-404-isvc-de145-predictor.18d62df202bedad7 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5b727472-03a2-4fc1-b5ee-6bf973753f72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 09f3fb27-c3af-4eaa-9db8-86e1f11850f0 kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: error-404-isvc-de145-predictor.18d62df90037cbae namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 4d59bb9d-b61c-4fbd-98e6-5e5b6272d69a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 09f3fb27-c3af-4eaa-9db8-86e1f11850f0 kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: error-404-isvc-de145-predictor.18d62df900aab080 namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: 84ffd3f0-52fa-4ee1-be61-62aec683b6f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de145 namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: 125cce07-e030-4ffa-b3b8-fa7f116ec437 kind: Event lastTimestamp: "2026-09-17T18:07:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:33Z" name: error-404-isvc-de145.18d62dee83d7e13f namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: f647cad8-4b78-45cc-b324-28a7bd49bfbb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de145 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 125cce07-e030-4ffa-b3b8-fa7f116ec437 kind: Event lastTimestamp: "2026-09-17T18:07:33Z" message: InferenceService [error-404-isvc-de145] is Ready metadata: creationTimestamp: "2026-09-17T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:33Z" name: error-404-isvc-de145.18d62df9dc808475 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: b4458109-68e8-4e79-be9b-a0fece4ef729 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:29.453499Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f13ae-predictor-776cc9696c-7jk49 to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:29Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d509d1d95 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: e0867084-8b17-47c7-892f-8d5c6b882138 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d7359f2de namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 946ac0da-f35f-49c0-8a86-7274a5c888fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d74e14bbd namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 5adb01e4-61f9-48c3-9b47-8a86eb4ad0c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d7850430f namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: d0a5ade9-eb5b-4e57-9808-1d2218079304 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d78c0c50f namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: a54ef8fa-8014-42a7-9f54-4fb0b504d02a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d78ca81c1 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: acee1080-b71d-440e-a83c-81f052abcbbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d7b758dab namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 81bb366f-f499-4b75-bde2-389f3dfd331e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d7be480f9 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 892138c8-ebcc-4179-992f-d245be3237c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T18:06:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:46Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62d6d88ce9ea3 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: adcdf47b-630e-40b0-b19d-9d9e195025a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62dee77a7329a namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 48ef406c-e6e9-4b74-9044-bcf376d16558 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62dee77a796fd namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 4f145b51-b3e9-492b-b886-bcef3d5dcf3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 8dfad140-1f0d-494d-825c-dd4e3391d778 kind: Event lastTimestamp: "2026-09-17T18:06:46Z" message: 'Readiness probe failed: Get "https://10.133.0.45:8643/healthz": dial tcp 10.133.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:46Z" name: error-404-isvc-f13ae-predictor-776cc9696c-7jk49.18d62deefd5e9f30 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: cb152165-4ca8-4a4c-93ce-39b11808e940 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f13ae-predictor-776cc9696c namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 40478374-ecf6-4f1b-8ccb-3c33590d077c kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: 'Created pod: error-404-isvc-f13ae-predictor-776cc9696c-7jk49' metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:29Z" name: error-404-isvc-f13ae-predictor-776cc9696c.18d62d6d500538fe namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 2e03ab52-d6bd-4996-a428-50cac211fe55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: d347b166-a5be-4ece-a67e-5135c7ffc711 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Scaled up replica set error-404-isvc-f13ae-predictor-776cc9696c from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:29Z" name: error-404-isvc-f13ae-predictor.18d62d6d4e337859 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: a233c6ce-e7f0-4291-9114-cf17c3aedf43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 51fcef2e-c199-4ac6-a769-726ec0101b6d kind: Event lastTimestamp: "2026-09-17T17:57:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:59Z" name: error-404-isvc-f13ae-predictor.18d62d70cca86eea namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: fcedd4e0-ac82-4bb8-b43d-cec30aae2ef0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 51fcef2e-c199-4ac6-a769-726ec0101b6d kind: Event lastTimestamp: "2026-09-17T17:57:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:59Z" name: error-404-isvc-f13ae-predictor.18d62d70ccf5ece4 namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 925add1c-8857-4e5c-a3f2-2d9c8b527bf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 51fcef2e-c199-4ac6-a769-726ec0101b6d kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:29Z" name: error-404-isvc-f13ae-predictor.18d62d77cb99a004 namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 8af36406-31e9-44ee-ac07-1ea8d9152480 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 51fcef2e-c199-4ac6-a769-726ec0101b6d kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:29Z" name: error-404-isvc-f13ae-predictor.18d62d77cc0412b5 namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 2f743c6d-de41-443c-8bb3-70388360928d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 8a2e219c-c19e-4af2-a358-2f6612746e87 kind: Event lastTimestamp: "2026-09-17T17:58:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:16Z" name: error-404-isvc-f13ae.18d62d6d4e387021 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 10070eb0-2550-45be-a053-be2d6fd621a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 8a2e219c-c19e-4af2-a358-2f6612746e87 kind: Event lastTimestamp: "2026-09-17T17:58:16Z" message: InferenceService [error-404-isvc-f13ae] is Ready metadata: creationTimestamp: "2026-09-17T17:58:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:16Z" name: error-404-isvc-f13ae.18d62d7844408681 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: bcb25966-0b56-4018-91f2-fa3dfa39ebc7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:44:19.380522Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q to ip-10-0-132-234.ec2.internal metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb55c983c01 namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: 8764a035-dbc7-4178-97e7-4f9d69aa8e96 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb56dfc3903 namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: 3cdd54af-a37f-422f-a396-1bb97d246dc0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb5a247a0f7 namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: b31b9310-aafa-4259-b36e-1b80afc4f553 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5647bbf3d2f892ce727c30831f3638ccee1c77720d8b8d5d0b09ee9d27787d0e" already present on machine metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb5a444ebb8 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: a71ab265-a45c-4a0b-a8bc-90734b218e3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb5a7316362 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 5965a728-354e-43fd-8cc6-39501c2d5e10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb5a7b65e88 namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 177bc7b6-81fd-46ac-8ba8-b078f18d3f3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:25Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1903" metadata: creationTimestamp: "2026-09-17T17:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:25Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cb6be9749b9 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 183c6e44-db95-44a8-a7a0-fd391741ea34 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1903" in 27.368s (27.368s including waiting). Image size: 1583073164 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbd1de8c9e0 namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: 3ef9ab3c-192f-47b0-8777-f16ccc6f1a85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbd20d87cce namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 8bcd16ce-01b5-4e80-8dd3-e02b67c73d3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbd21351834 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: d1897dcd-5235-4d61-9d70-976f73c91e19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbd2143f413 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: f8f9f915-7d67-4400-a4de-6a2955246d98 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.773s (2.773s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbdc68d7e17 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 5f9d522a-9fbb-4a4e-9f0d-498a11a59464 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbdcb673566 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 7609495d-bdf1-4847-ab28-03a8cba47372 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbdcbfe25f4 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 17b47d96-779d-4fd7-b590-3871dff865e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cbe4077b93a namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: cf073245-a480-48bc-972c-a816a93a4d69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cd86cbd6027 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 75fc44b4-21a9-4e09-9137-c0a4bc93c3da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cd86cbdde9d namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: e1f73f57-b2ab-48d0-9e95-7e69da557093 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f45bf1e-041c-4733-928c-7f6056a0b354 kind: Event lastTimestamp: "2026-09-17T17:46:53Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:53Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q.18d62cd942d75621 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: c2151830-d9e6-4010-a73b-aed62b99ab4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7b5b9cf66f namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: ebf3bd88-218b-4cdf-8b15-86e660ec4f5d kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7b5b9cf66f-t5p8q' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-1-predictor-7b5b9cf66f.18d62cb55c1af7fc namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: 0bb93954-1aa7-48ae-a00a-f0e21409fb13 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: ca40da44-05f1-436f-a03c-405d012236ff kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7b5b9cf66f from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-1-predictor.18d62cb5545a7a20 namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 893e5af2-b981-4052-815c-ab19cf8f8338 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: 62a96e97-47b2-4266-8c43-6406aa8611ae kind: Event lastTimestamp: "2026-09-17T17:45:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:04Z" name: isvc-sklearn-graph-1-predictor.18d62cb8da7a9d66 namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 55b831d5-2941-4797-85c7-000daaa887ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: 62a96e97-47b2-4266-8c43-6406aa8611ae kind: Event lastTimestamp: "2026-09-17T17:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:04Z" name: isvc-sklearn-graph-1-predictor.18d62cb8db021858 namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: ef99e3bb-ece3-4998-89b2-0e884db3373a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 62a96e97-47b2-4266-8c43-6406aa8611ae kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-1-predictor.18d62cc357ced6b2 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 53c91075-fce9-4f2d-845b-3b89675294ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 62a96e97-47b2-4266-8c43-6406aa8611ae kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-1-predictor.18d62cc35865df04 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: a21531c5-44fb-4d23-a043-732fa64111c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 3caf625e-d3ee-4db8-8ce3-60550ba1a2a9 kind: Event lastTimestamp: "2026-09-17T17:46:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:11Z" name: isvc-sklearn-graph-1.18d62cb55a7e1a9d namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 0f20982d-0b7c-44d2-9f84-3845d5334208 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 3caf625e-d3ee-4db8-8ce3-60550ba1a2a9 kind: Event lastTimestamp: "2026-09-17T17:46:10Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-17T17:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:10Z" name: isvc-sklearn-graph-1.18d62ccf5915abeb namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 1f4fe896-b442-40ae-bf2f-306c721c88d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:44:19.898311Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 to ip-10-0-132-234.ec2.internal metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb57b751294 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 61d40c79-594a-4e0e-820f-f6ad3d554193 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb59d5fa1dc namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: b15f6664-48fa-4168-a9ab-de46591f929e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5647bbf3d2f892ce727c30831f3638ccee1c77720d8b8d5d0b09ee9d27787d0e" already present on machine metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb59f106d74 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 921c17dd-ff5c-4621-ad5e-8ebc0f73a6c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb5a26deca2 namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: abfa493d-d78b-46eb-93b4-2364ea8ba03a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb5a2e8a8da namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: bb95c0f0-a027-4ba0-b44d-d23f2d6b5765 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:24Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1903" metadata: creationTimestamp: "2026-09-17T17:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:24Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cb6818b1290 namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 05cf1b60-2999-4e50-b22f-b4d74e84cda7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1903" in 28.372s (28.372s including waiting). Image size: 1583073164 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbd1caef813 namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 339fbf98-92e9-433f-af6d-d9c1252a9740 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbd20c79a0a namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 0e316073-783d-42d4-90fe-42c50340a880 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbd213063c2 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: afd6959d-2e78-47fa-913e-dc6fd99ab3a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbd2143ef23 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: fb25ec64-7bed-487f-985c-46040dc6cd5a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.808s (2.808s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbdc8a437a3 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 7d5d93f0-78d6-4fc5-9ee9-7c21f98d8d06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbdcc4563af namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 4a510df6-88e8-4a07-b067-dde619844768 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbdccb8f4c9 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: ee1c00da-8404-4bf6-b13e-7645593d2549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T17:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cbe0492777a namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 9dfb76cf-a622-426e-ad61-62a8aa5c9801 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cd8715f0e7a namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 0090d2df-1a7b-4d60-a469-a3254ce89a31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cd871602cfc namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 8319ec15-ae81-4c12-b3ac-ee5c031823a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: b00b303b-6e83-445b-8bbc-4b643ede489c kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4.18d62cd906f3072a namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 09271bf8-5724-444a-9fb3-caf54d70f47e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-77459fbd8b namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 812c9fab-37e5-4b75-a6b2-a7338048936f kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-77459fbd8b-vtdt4' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-2-predictor-77459fbd8b.18d62cb57b089651 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 1b6fc6ef-c536-4b91-8d45-3392da8bc9ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: fe0de371-cc40-4981-9ff6-9ae6104d2c9a kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-77459fbd8b from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-sklearn-graph-2-predictor.18d62cb577ebd79f namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 11f1b4b5-adda-49b4-9e8a-4fcb448ad057 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 1f17d6a7-22f4-4579-a236-8b10639f6e55 kind: Event lastTimestamp: "2026-09-17T17:45:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:04Z" name: isvc-sklearn-graph-2-predictor.18d62cb8f66865e3 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 3ba7839d-dec9-4bea-986d-0dfd1ceb3f09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 1f17d6a7-22f4-4579-a236-8b10639f6e55 kind: Event lastTimestamp: "2026-09-17T17:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:04Z" name: isvc-sklearn-graph-2-predictor.18d62cb8f6cf8ef4 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 1d7d7609-03a9-4600-aed9-0bc62cbec470 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 1f17d6a7-22f4-4579-a236-8b10639f6e55 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-2-predictor.18d62cc372edf7a3 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 5034fa5b-c436-418e-b7eb-7dcba09b8972 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 1f17d6a7-22f4-4579-a236-8b10639f6e55 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: isvc-sklearn-graph-2-predictor.18d62cc37363ddde namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 69321697-eeb0-4d7f-a9f6-5c183bb67326 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: ede664c1-e9da-4a57-be87-ebd2758e31c3 kind: Event lastTimestamp: "2026-09-17T17:46:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:12Z" name: isvc-sklearn-graph-2.18d62cb577f34952 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 6dbf2003-6548-407c-8058-e7d8e7548e35 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: ede664c1-e9da-4a57-be87-ebd2758e31c3 kind: Event lastTimestamp: "2026-09-17T17:46:11Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-17T17:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:11Z" name: isvc-sklearn-graph-2.18d62ccf94aa2b08 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: bcc372c5-baba-4077-8bf2-7e1778858c6e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:44:19.676379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-ttnbx to ip-10-0-132-234.ec2.internal metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:44:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb56e3a887c namespace: kserve-ci-e2e-test resourceVersion: "14199" uid: 2efa802a-0377-4b9a-8833-87f063427a34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:44:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb58ff0f546 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: b9969498-dce4-432a-aa17-28df3dd6c8d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5647bbf3d2f892ce727c30831f3638ccee1c77720d8b8d5d0b09ee9d27787d0e" already present on machine metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb59196b0b6 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: aa2bc866-694e-4171-af34-fb1d4a96df2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb59484c1c0 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 594c26be-1c73-45d3-98a2-aa4845a4a2e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb594fd606f namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: cf2e1eb5-6d22-48a1-99c7-57b1198c65cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:24Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-17T17:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:24Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cb68173d8d4 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 224ac722-7010-4b50-b469-875e965273b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.471s (28.471s including waiting). Image size: 1435742979 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbd228187de namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 4c061f03-b2c5-479d-ad88-b34aa9295ad2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbd26bf07ae namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 1befb505-6bfa-4e8d-9300-abbe66833038 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbd2744cd1c namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: b9d060f4-1bd0-4af0-9db5-b27358c5dff1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbd27514db2 namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: 1635db0d-80bd-488c-9c4d-4f9a13720c95 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.721s (2.721s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbdc982e11b namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: afc38cba-b47f-4631-9913-dcf279423b1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbdcc654180 namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: a0708794-003b-4de6-a90f-a3827dba962b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:44:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbdccd74c42 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 27671465-c30f-4016-8491-7f512c6da6cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cbe0475985c namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: a5402f8c-29bb-445f-acf9-9007d357a205 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cd86eddf4d2 namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 65fbf8ab-09f1-44b7-bb2a-580cdd826624 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cd86ede4645 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 98be30f1-9399-4ca5-a997-ec29a3a7718a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 52346f4a-4e00-400b-8a00-412f1a2df659 kind: Event lastTimestamp: "2026-09-17T17:46:52Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ttnbx.18d62cd906f1c18f namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 44d2790a-61f4-44d0-a265-50702b151704 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-234.ec2.internal source: component: kubelet host: ip-10-0-132-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 2a6d243f-5f7b-4878-8f8a-4b19d1c48285 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-ttnbx' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d62cb56da5f16e namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 4af5fa19-74e4-438f-8910-87d4d8a286be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 2b1f986d-eaa9-40cf-a154-8d10eea705af kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: isvc-xgboost-graph-predictor.18d62cb5671a15f4 namespace: kserve-ci-e2e-test resourceVersion: "14185" uid: 75194ee9-8bb1-405a-a59a-43fd64cad945 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: 22360017-fc0e-4779-9222-f30222d6098c kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: isvc-xgboost-graph-predictor.18d62cb8e5b44590 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: e1528a31-1b86-4031-a8e9-51ecc3bc6ccf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: 22360017-fc0e-4779-9222-f30222d6098c kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: isvc-xgboost-graph-predictor.18d62cb8e6204e35 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 5d5dc8b7-c96c-4e91-9b07-324232629251 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 22360017-fc0e-4779-9222-f30222d6098c kind: Event lastTimestamp: "2026-09-17T17:46:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:34Z" name: isvc-xgboost-graph-predictor.18d62cc3625b9169 namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 10c2ff32-95cd-4a84-8379-a1f5b54e5ece reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 22360017-fc0e-4779-9222-f30222d6098c kind: Event lastTimestamp: "2026-09-17T17:46:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:34Z" name: isvc-xgboost-graph-predictor.18d62cc362cff0a7 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 373f56b9-7ad6-47d1-a016-ea0cfcc0e386 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: 46287ae8-8310-47b6-9e80-ef7dae0bca34 kind: Event lastTimestamp: "2026-09-17T17:46:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:02Z" name: isvc-xgboost-graph.18d62cb567293f1e namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 0b68e1f8-8623-471a-a23c-341eb14d1bbb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 46287ae8-8310-47b6-9e80-ef7dae0bca34 kind: Event lastTimestamp: "2026-09-17T17:46:02Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-17T17:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:02Z" name: isvc-xgboost-graph.18d62ccd6b20a318 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 293121c8-d1eb-4fa8-90f9-77945183fab3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:39.243276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7cf9877cc4-6tdsb to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:39Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd5ed1072c9 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: b050e0b6-b2ed-4347-b45b-45df23c079b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:39Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd5f8e53319 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: d5aa7b0d-88ff-47fe-bdb5-a8edf7cc6f69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:40Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:40Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd6328eac41 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 00415e87-906b-4ee1-a9e6-2c75c725a348 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:40Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd633c9acce namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: b97486de-3b08-4e2b-9fbb-94df2bdf1e42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:40Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-17T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:40Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd6368bb421 namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: 25eb7001-eb37-4807-aca8-34052322ddf7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:40Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-17T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:40Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd636f4c143 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: a08f926c-dd3f-441d-b387-fb316da0512d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:49Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd8676c3934 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: e61cdcc5-9679-4db3-abde-d275ceb18062 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7cf9877cc4-6tdsb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0b691b26-f64b-4c40-8403-f5e4aec2cfc1 kind: Event lastTimestamp: "2026-09-17T17:47:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:17Z" name: model-chainer-7cf9877cc4-6tdsb.18d62cd8f7b047de namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 0f6344b3-85c7-4e54-9a36-1814ee7e1945 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7cf9877cc4 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 23c8e789-5979-48be-ab6d-1c1878a14b5e kind: Event lastTimestamp: "2026-09-17T17:46:39Z" message: 'Created pod: model-chainer-7cf9877cc4-6tdsb' metadata: creationTimestamp: "2026-09-17T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:39Z" name: model-chainer-7cf9877cc4.18d62cd5ec452856 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 3e4cfc79-457d-4032-a9e9-c55b3ce0297b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 9be2b722-e903-4c5c-8ab1-83048a1663a5 kind: Event lastTimestamp: "2026-09-17T17:46:39Z" message: Scaled up replica set model-chainer-7cf9877cc4 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:39Z" name: model-chainer.18d62cd5eb3a8649 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 1cec643d-2956-43af-a249-efdc6ca06f32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f7ebd8d9-0288-45f7-8ee1-c3a0c9482e0c kind: Event lastTimestamp: "2026-09-17T17:46:39Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-17T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:39Z" name: model-chainer.18d62cd5f111b988 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 90220573-452a-421b-ae48-2920a7df7235 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:47:50.046500Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-390a8-857fc7d7c6-qzhnq to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce66943ece9 namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: b5860bf2-887e-41e9-a05d-e25db9ea1d85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-390a8-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce674806835 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: c2e09e63-f0c6-4800-909c-6726ed7658ab reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:47:51Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:47:51Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce6aef784a5 namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: c0782939-0681-4426-80fb-5bd807c569ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-390a8} kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:47:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:51Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce6b021e2fb namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: 421e35ea-7c54-4797-ac92-3f6807538344 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-390a8} kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:47:51Z" message: 'Created container: sequence-graph-390a8' metadata: creationTimestamp: "2026-09-17T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:51Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce6b2f53c86 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 51a276a4-28fa-4c6e-82c0-5709eb750d53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-390a8} kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:47:51Z" message: Started container sequence-graph-390a8 metadata: creationTimestamp: "2026-09-17T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:51Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62ce6b3676986 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 498676c0-8108-43fb-a7ec-cdb5de552778 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-390a8} kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Stopping container sequence-graph-390a8 metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:04Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62d599938f5b1 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: e1e5d6ce-41db-4aaf-9a37-6374c19d34f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-390a8} kind: Pod name: sequence-graph-390a8-857fc7d7c6-qzhnq namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 14121ca3-b64b-446c-aa51-106ab342feaa kind: Event lastTimestamp: "2026-09-17T17:56:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:32Z" name: sequence-graph-390a8-857fc7d7c6-qzhnq.18d62d5a4077ea6f namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 90a89553-c7d6-4947-bf6d-4e796b00017f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-390a8-857fc7d7c6 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 1fa411b1-1204-43a3-b7d5-57375ad94a6c kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: 'Created pod: sequence-graph-390a8-857fc7d7c6-qzhnq' metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8-857fc7d7c6.18d62ce668d52a8d namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 156e454a-475b-425d-bd55-ed74d388df63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 7b3a6b0e-968f-4254-8fe4-dff2db08e81c kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: Scaled up replica set sequence-graph-390a8-857fc7d7c6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8.18d62ce667e9dbaf namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 631f7646-cfc1-47dc-85ca-2971430c4d60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 97f08d27-98a7-45f6-8bf6-1737a04658fa kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: InferenceGraph [sequence-graph-390a8] is Ready metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8.18d62ce66d78444a namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: aa689cd1-0f35-4a66-86f5-6c85917f1ffc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 97f08d27-98a7-45f6-8bf6-1737a04658fa kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: 'Failed to update status for InferenceGraph "sequence-graph-390a8": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-390a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8.18d62ce6706502c2 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: dc28792a-f961-4d60-b463-813fa609bdd6 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 97f08d27-98a7-45f6-8bf6-1737a04658fa kind: Event lastTimestamp: "2026-09-17T17:47:50Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-390a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:50Z" name: sequence-graph-390a8.18d62ce6706543f5 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 2a836a9d-8b52-4b1a-97c1-c41542eaf3a3 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: fb3d0607-3714-4da3-a4ba-b7a9616f04dd kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:05Z" name: sequence-graph-390a8.18d62ce9e64f8745 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: adbcf401-3763-466d-804c-eddba9ce9117 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: fb3d0607-3714-4da3-a4ba-b7a9616f04dd kind: Event lastTimestamp: "2026-09-17T17:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:05Z" name: sequence-graph-390a8.18d62ce9e6aa7030 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 42a1924a-9b10-4c70-a1d1-7053e3c55170 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: fb3d0607-3714-4da3-a4ba-b7a9616f04dd kind: Event lastTimestamp: "2026-09-17T17:48:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:20Z" name: sequence-graph-390a8.18d62ced6563b971 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: eb5be902-6ae5-48ae-af49-db5d2aed1170 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: fb3d0607-3714-4da3-a4ba-b7a9616f04dd kind: Event lastTimestamp: "2026-09-17T17:48:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:20Z" name: sequence-graph-390a8.18d62ced65d2f586 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 1ab33dbb-7993-4061-a49a-d7d2e94ae1d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:14.995571Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:14Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d69f2da919c namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 59fe4acd-dbf2-4624-959b-8b124e4c3f91 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6b6c1-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:15Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d69ff783cab namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 1ef63c0f-4fa1-4539-823e-2879da529cc0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:16Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:16Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6a38d17754 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 50175087-bd13-4bab-b3ed-ebcdd0a8796e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b6c1} kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:16Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6a3a26d4cd namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: b4da98cb-6ab2-459e-a29a-867c76fef0e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b6c1} kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:16Z" message: 'Created container: sequence-graph-6b6c1' metadata: creationTimestamp: "2026-09-17T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:16Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6a3cfde3b2 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 220b9c2b-d427-480e-96a7-a7aac0670bb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b6c1} kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:16Z" message: Started container sequence-graph-6b6c1 metadata: creationTimestamp: "2026-09-17T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:16Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6a3d6a97bd namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 0947976a-4cbb-4398-829a-7d8429379d86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b6c1} kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container sequence-graph-6b6c1 metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6d3ac73b25 namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 0fe06c85-0f2a-4d70-bb22-d7d4a9c508b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b6c1} kind: Pod name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: d23eee3a-d98a-4848-9877-9f051241ce51 kind: Event lastTimestamp: "2026-09-17T17:57:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:57Z" name: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6.18d62d6e1bced1cb namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 438c8c9f-aafc-4dec-8ccc-5a21f58c76bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6b6c1-7cb7fbb4f9 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 5b7523c4-6d62-4a77-8481-68658cda0767 kind: Event lastTimestamp: "2026-09-17T17:57:14Z" message: 'Created pod: sequence-graph-6b6c1-7cb7fbb4f9-xm9z6' metadata: creationTimestamp: "2026-09-17T17:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:14Z" name: sequence-graph-6b6c1-7cb7fbb4f9.18d62d69f2641ce1 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 01bbaf41-74f0-4fa5-8db9-7481077ee54b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 54062a63-84db-4804-843b-17d6353de7e8 kind: Event lastTimestamp: "2026-09-17T17:57:14Z" message: Scaled up replica set sequence-graph-6b6c1-7cb7fbb4f9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:14Z" name: sequence-graph-6b6c1.18d62d69f1761fa1 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 06181ba5-1160-41e4-ac39-1a95c636ca9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 69e8bdf5-a9e5-42f9-a5c4-0f83cdf49b74 kind: Event lastTimestamp: "2026-09-17T17:57:15Z" message: InferenceGraph [sequence-graph-6b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:15Z" name: sequence-graph-6b6c1.18d62d69f9e6937b namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 1ff5d614-ce63-4a40-8ce5-2d2a49472750 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:58:29.345723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f13ae-d54cbf9fb-xkjfq to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:58:29Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b4277e989 namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: ce2f69ff-8997-4745-bd81-9600286ef9e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f13ae-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:29Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b4c6dab9b namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 4197b1a6-5200-4e15-a51e-61863d955416 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:58:30Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b8863d3db namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: ca72893a-24b5-4951-af6b-a1b6406dfe65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f13ae} kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b89acb67b namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f16562de-20b9-45ce-adeb-70a3af1d12bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f13ae} kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: 'Created container: sequence-graph-f13ae' metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b8cb8ee9e namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 8552f820-3295-496c-a9ac-4ed92777b513 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f13ae} kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T17:58:30Z" message: Started container sequence-graph-f13ae metadata: creationTimestamp: "2026-09-17T17:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:58:30Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62d7b8d25405e namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: 8bb3c210-a67d-4de2-bae9-6b68025755d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f13ae} kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container sequence-graph-f13ae metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62dee6cc26ece namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: 5a1e3901-4fe3-489a-a347-6d6499e91922 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f13ae} kind: Pod name: sequence-graph-f13ae-d54cbf9fb-xkjfq namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 50e84966-d1db-4ed5-92b5-0e3f25094d75 kind: Event lastTimestamp: "2026-09-17T18:07:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:12Z" name: sequence-graph-f13ae-d54cbf9fb-xkjfq.18d62def2b72f283 namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: 206cebb5-9f66-42e3-820b-174a7986f5d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f13ae-d54cbf9fb namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 0bf55edd-319d-460a-9708-55bed7d4dbc4 kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: 'Created pod: sequence-graph-f13ae-d54cbf9fb-xkjfq' metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:29Z" name: sequence-graph-f13ae-d54cbf9fb.18d62d7b42032700 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 54f0a095-eff1-41e4-9ea2-773997cab04d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 6e98ce8e-7638-4947-9d51-94d80343d36d kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: Scaled up replica set sequence-graph-f13ae-d54cbf9fb from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:29Z" name: sequence-graph-f13ae.18d62d7b40b681a0 namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 18a0b483-164c-4b04-9cbc-eec9e4a336a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: 25de2291-ff62-4748-a780-61c916a582fa kind: Event lastTimestamp: "2026-09-17T17:58:29Z" message: InferenceGraph [sequence-graph-f13ae] is Ready metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:29Z" name: sequence-graph-f13ae.18d62d7b4795c0de namespace: kserve-ci-e2e-test resourceVersion: "21947" uid: 2a4a8d4f-bfd1-4f99-979c-f7523b367d21 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: ba2dacfe-689b-4b1c-9d85-7b385ec69461 kind: Event lastTimestamp: "2026-09-17T17:58:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:59Z" name: sequence-graph-f13ae.18d62d7ebf4ea436 namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: 537a2b00-113f-438e-836c-9de0bb6d9b8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: ba2dacfe-689b-4b1c-9d85-7b385ec69461 kind: Event lastTimestamp: "2026-09-17T17:58:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:59Z" name: sequence-graph-f13ae.18d62d7ebf9c1bb3 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: e5828b82-33ec-45e1-ab6d-faf04212053d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:07:43.931212Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-3ef6b-6679565d6c-hfv7z to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T18:07:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:07:43Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62dfc6256dbda namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: 03206dff-16c1-4566-be0d-db4b193df951 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:44Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:07:44Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:07:44Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62dfc84372733 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: d5e90267-66a9-4144-93e4-cc93965d8271 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3ef6b} kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:07:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:44Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62dfc859d1bb7 namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 6f002ef0-0d86-40af-9035-88db0b74d32f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3ef6b} kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:07:44Z" message: 'Created container: splitter-graph-3ef6b' metadata: creationTimestamp: "2026-09-17T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:44Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62dfc8871438f namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: ab27a4ca-1499-4646-8c56-c1b386524506 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3ef6b} kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:07:44Z" message: Started container splitter-graph-3ef6b metadata: creationTimestamp: "2026-09-17T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:44Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62dfc88dc695d namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: b107629e-c1d3-4cbc-99cd-e69e5f4acb6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:15:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3ef6b} kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:15:58Z" message: Stopping container splitter-graph-3ef6b metadata: creationTimestamp: "2026-09-17T18:15:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:15:58Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62e6f8b2f25d1 namespace: kserve-ci-e2e-test resourceVersion: "30295" uid: 9ac27a35-8217-4d47-8f77-611b369446d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3ef6b} kind: Pod name: splitter-graph-3ef6b-6679565d6c-hfv7z namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 23560048-9ee0-4665-a35a-d3feee8ad2e4 kind: Event lastTimestamp: "2026-09-17T18:16:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:16:26Z" name: splitter-graph-3ef6b-6679565d6c-hfv7z.18d62e7032a9250a namespace: kserve-ci-e2e-test resourceVersion: "30529" uid: 61862404-dbee-4c48-9089-a5866dd5930c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-3ef6b-6679565d6c namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: c05f1251-a9ae-4fee-a791-b9fdd032550f kind: Event lastTimestamp: "2026-09-17T18:07:43Z" message: 'Created pod: splitter-graph-3ef6b-6679565d6c-hfv7z' metadata: creationTimestamp: "2026-09-17T18:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:43Z" name: splitter-graph-3ef6b-6679565d6c.18d62dfc619fd27d namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: aac529ef-5a99-4478-a018-263bf49d6960 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: f4b9750d-18ff-4d69-9d65-0485623fc016 kind: Event lastTimestamp: "2026-09-17T18:07:43Z" message: Scaled up replica set splitter-graph-3ef6b-6679565d6c from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:43Z" name: splitter-graph-3ef6b.18d62dfc6080f0be namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: 96dbd8ec-d32d-41f8-a843-183994d317d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: a0c8830a-b095-4efc-84bc-759ed78b6708 kind: Event lastTimestamp: "2026-09-17T18:07:44Z" message: InferenceGraph [splitter-graph-3ef6b] is Ready metadata: creationTimestamp: "2026-09-17T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:44Z" name: splitter-graph-3ef6b.18d62dfc66763f2b namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 675732c8-9891-409a-820d-632fc43fffdf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 57ce35dd-6e52-4f89-af15-02436ec492c4 kind: Event lastTimestamp: "2026-09-17T18:07:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:58Z" name: splitter-graph-3ef6b.18d62dffdee14bea namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: e9e45903-d5b6-439d-9824-d58cfd22600d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 57ce35dd-6e52-4f89-af15-02436ec492c4 kind: Event lastTimestamp: "2026-09-17T18:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:58Z" name: splitter-graph-3ef6b.18d62dffdf49bbfc namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: f028fd9b-3f4f-4deb-b133-0826ccbd4095 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 57ce35dd-6e52-4f89-af15-02436ec492c4 kind: Event lastTimestamp: "2026-09-17T18:08:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:13Z" name: splitter-graph-3ef6b.18d62e035dee345b namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: cd33b4da-d7f7-4f9d-9604-892a0626a285 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 57ce35dd-6e52-4f89-af15-02436ec492c4 kind: Event lastTimestamp: "2026-09-17T18:08:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:13Z" name: splitter-graph-3ef6b.18d62e035e43d03e namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: 2dbfa1c3-e790-4b7d-96d5-866d15357afe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:06:33.593672Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25509" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b5897-5bdb7c67d9-65869 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T18:06:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:06:33Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dec01e52e55 namespace: kserve-ci-e2e-test resourceVersion: "25515" uid: 1ef97709-c824-4d64-995f-08d6e1f89fff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:34Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:06:34Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:06:34Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dec23bb4cdb namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 9df16912-88c4-440d-ace0-e5c35cd28bba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5897} kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:06:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:34Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dec250eabc7 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 7df810d5-7c24-482c-919a-a4b6dd507982 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5897} kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:06:34Z" message: 'Created container: splitter-graph-b5897' metadata: creationTimestamp: "2026-09-17T18:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:34Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dec27afad2b namespace: kserve-ci-e2e-test resourceVersion: "25537" uid: e8836409-a5ac-49fe-8087-0ff7901f50ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5897} kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:06:34Z" message: Started container splitter-graph-b5897 metadata: creationTimestamp: "2026-09-17T18:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:34Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dec281c368a namespace: kserve-ci-e2e-test resourceVersion: "25538" uid: fec0ae6a-13a8-4e4a-a6f8-13d4be4c40f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5897} kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container splitter-graph-b5897 metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62dee5adc8da5 namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: e0ccce83-0bb0-4bdf-8e72-99f15c974e6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5897} kind: Pod name: splitter-graph-b5897-5bdb7c67d9-65869 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 72dc6a53-ed0d-4407-b9ab-2259ad6d6c04 kind: Event lastTimestamp: "2026-09-17T18:07:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:11Z" name: splitter-graph-b5897-5bdb7c67d9-65869.18d62deee94c511f namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 25941464-355d-4c9d-bc67-e197239cb0ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b5897-5bdb7c67d9 namespace: kserve-ci-e2e-test resourceVersion: "25506" uid: ff24257a-7dfe-48e6-96b1-edcbd4e8c2d0 kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: 'Created pod: splitter-graph-b5897-5bdb7c67d9-65869' metadata: creationTimestamp: "2026-09-17T18:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: splitter-graph-b5897-5bdb7c67d9.18d62dec0167b366 namespace: kserve-ci-e2e-test resourceVersion: "25512" uid: ac200d7f-10f8-4936-9bef-2f6df0ba0f05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b5897 namespace: kserve-ci-e2e-test resourceVersion: "25504" uid: bdaa8cac-e8f8-4932-b517-9bec6c5809e2 kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: Scaled up replica set splitter-graph-b5897-5bdb7c67d9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: splitter-graph-b5897.18d62dec003faf41 namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: 5659c3d9-6a2e-4d7d-979c-9c668b256d80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b5897 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 93e5d553-c3d1-448a-b189-cd8079197bc4 kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: InferenceGraph [splitter-graph-b5897] is Ready metadata: creationTimestamp: "2026-09-17T18:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:33Z" name: splitter-graph-b5897.18d62dec073bec32 namespace: kserve-ci-e2e-test resourceVersion: "25527" uid: 58649392-b07a-4886-b368-af516ca6d095 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:55:04.365221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4b88adabdb namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: 830141e1-8bda-4eed-aa48-339d26615609 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19660" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bab15b3b5 namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 30c852ff-2b74-40f9-9445-54efbc32d7c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4baca41d05 namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: c4ae28bc-6a93-48b0-ba18-f8e55613928e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bafa89a54 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 06d4e6d5-aeed-4638-8ff2-309a0267f4b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bb01c1bbf namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: cdac0d0f-356f-45ab-aba0-fe607cea0e23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bb0278e66 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 8e0168d0-3551-40a0-b1d3-d5d5e7f8ea3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bb2c9c2e2 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: b052ee12-21f1-4e13-8458-7f354a303fb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:05Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bb3388bac namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 64dc1352-6aea-4ab8-8222-90bd0038a134 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:56:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:21Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d4bd70aefd6 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: ed0f3ce0-834c-4bad-a343-6a0bb4bb145b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d5cd05283a5 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: e0b3debd-bc41-4132-8e8f-9285904866c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d5cd052e909 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 735ed7c1-7ac2-4bb1-87dd-6c8526e84b71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 64036856-d29a-4195-ba9d-dcd53de9cd4a kind: Event lastTimestamp: "2026-09-17T17:56:21Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:21Z" name: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp.18d62d5d89624cd9 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 209d6c8d-1a93-4714-a128-7ff94e02714f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2b6c1-predictor-65f5775b6 namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: 7274b509-87fb-4161-9748-76a2573ccf70 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'Created pod: success-200-isvc-2b6c1-predictor-65f5775b6-6s9hp' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1-predictor-65f5775b6.18d62d4b88275133 namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: ac74ffd4-f801-4433-a350-c873464c3e3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19620" uid: 2ed1bf8d-fec8-4b0c-b527-8b22e32c3d26 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Scaled up replica set success-200-isvc-2b6c1-predictor-65f5775b6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1-predictor.18d62d4b843fd1bb namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 8ce8ecf2-727e-4f1f-87e5-a81b10355088 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: cedb93c0-cbab-4b50-8bfe-0e1454cc2657 kind: Event lastTimestamp: "2026-09-17T17:55:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:34Z" name: success-200-isvc-2b6c1-predictor.18d62d4f02f3bdf8 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 14a7df06-1639-4e04-a0e4-ef5c72341df0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: cedb93c0-cbab-4b50-8bfe-0e1454cc2657 kind: Event lastTimestamp: "2026-09-17T17:55:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:55:34Z" name: success-200-isvc-2b6c1-predictor.18d62d4f034c9ed5 namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: 0727830e-c5e5-433b-a106-2d586aea88c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: cedb93c0-cbab-4b50-8bfe-0e1454cc2657 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-2b6c1-predictor.18d62d56021ef078 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 6d341d15-b25c-47d6-982a-20a4cacb27a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: cedb93c0-cbab-4b50-8bfe-0e1454cc2657 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-2b6c1-predictor.18d62d560270b00d namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: a9bab9bd-2ffb-417e-ba94-af95c728ff34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 6ffa7aa4-7dfe-4eab-9581-f6be5d167aa5 kind: Event lastTimestamp: "2026-09-17T17:55:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:51Z" name: success-200-isvc-2b6c1.18d62d4b8480298e namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 8128b2b2-63f9-4cf0-8308-6a1556cbf8c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 6ffa7aa4-7dfe-4eab-9581-f6be5d167aa5 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-2b6c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1.18d62d4b84fe2007 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 07906e00-8090-4706-8ac5-17027a12354a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 6ffa7aa4-7dfe-4eab-9581-f6be5d167aa5 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2b6c1.18d62d4b84fe839e namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 3cac87c1-44cc-48c9-8699-b03a25b3b74a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 6ffa7aa4-7dfe-4eab-9581-f6be5d167aa5 kind: Event lastTimestamp: "2026-09-17T17:55:51Z" message: InferenceService [success-200-isvc-2b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:55:51Z" name: success-200-isvc-2b6c1.18d62d56944a7a8b namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 83a17759-53ae-4a85-bbfa-3970ef394c8f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:45:49.703470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2d039-predictor-58dcb9dff4-h64gn to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:45:49Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca64430855 namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 443ee152-1f5a-4e71-9d20-b085c52d2355 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca869a41b9 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 0d17a1b1-1c2f-4613-9539-9f7f3c1ce77a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca882e3372 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 8b4898a7-98ae-41d6-9402-58e33f4bedcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca8b33d926 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 6b8342d5-379c-4624-8f14-62ed82888e2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca8bb69551 namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 8a647c5f-ed83-49ae-90ef-611e57a326c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca8bbf3ff2 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: ec67eb7e-08f1-4fde-b571-3ad112862315 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca8ea02976 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 0b0902bc-ba6e-4937-a3d4-e701085d19cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:45:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:50Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62cca8f0f7c6a namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 3105c7af-183d-4b15-8d0a-274760b9cd01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:46:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:28Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62ccaf6c7bba2 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: b54551cb-6cc2-4aa5-88a7-0dc57aab2004 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62d4b82dc8a59 namespace: kserve-ci-e2e-test resourceVersion: "19616" uid: 96dfa02b-4513-4c44-bcb1-7df2bb314aa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: a9696747-1e5f-41ed-9a1a-d6b0d76c6a16 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn.18d62d4b82dd0e3b namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: 093c4729-bd7a-40d2-8621-a333a57e4e8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2d039-predictor-58dcb9dff4 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 72444de9-28f8-4ebb-aae1-5580f97a8650 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: 'Created pod: success-200-isvc-2d039-predictor-58dcb9dff4-h64gn' metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:49Z" name: success-200-isvc-2d039-predictor-58dcb9dff4.18d62cca63d640ab namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 37b673a7-4709-48c5-b505-ca6dd43535b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: c7cc95a8-607b-40ca-9128-4c0d84ea34bc kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Scaled up replica set success-200-isvc-2d039-predictor-58dcb9dff4 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:49Z" name: success-200-isvc-2d039-predictor.18d62cca61868bcf namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 75b0ea7a-ba35-422b-a1d9-883e1b4c0680 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: a0c04b11-135e-4a3d-8a70-bcdfac6f80fb kind: Event lastTimestamp: "2026-09-17T17:46:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:19Z" name: success-200-isvc-2d039-predictor.18d62ccde007db71 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 081a03e8-6de8-4869-b524-6a900d1cdd69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: a0c04b11-135e-4a3d-8a70-bcdfac6f80fb kind: Event lastTimestamp: "2026-09-17T17:46:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:19Z" name: success-200-isvc-2d039-predictor.18d62ccde07b392c namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 2b40b3be-f131-4cf7-bef2-315687c75763 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: a0c04b11-135e-4a3d-8a70-bcdfac6f80fb kind: Event lastTimestamp: "2026-09-17T17:47:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:04Z" name: success-200-isvc-2d039-predictor.18d62cd4ddc88c0c namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: a07f2051-344a-4274-82f8-4eb29fe04a5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d039-predictor namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: a0c04b11-135e-4a3d-8a70-bcdfac6f80fb kind: Event lastTimestamp: "2026-09-17T17:47:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:04Z" name: success-200-isvc-2d039-predictor.18d62cd4de482501 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 41a7687c-ab91-419a-af95-f1223b8663aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: bcde812e-341e-45d8-b07c-81f748d806c9 kind: Event lastTimestamp: "2026-09-17T17:46:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:38Z" name: success-200-isvc-2d039.18d62cca619c449b namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 19332cf2-f2af-4b70-9ade-125aa6029249 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: bcde812e-341e-45d8-b07c-81f748d806c9 kind: Event lastTimestamp: "2026-09-17T17:46:38Z" message: InferenceService [success-200-isvc-2d039] is Ready metadata: creationTimestamp: "2026-09-17T17:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:38Z" name: success-200-isvc-2d039.18d62cd5b337143a namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 724349a6-1b03-46b4-bb10-0cef5eee1caa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:50.155348Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-390a8-predictor-6749bbb468-v5k5t to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:50Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd877796986 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: ddb8e30f-4c16-4105-9915-6c98bbf0b7f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-390a8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd887b05e97 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 5ad4817b-5cce-48e5-94ad-308c62c15476 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8bd5f4e64 namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8356b9a0-528e-4b7e-b45a-030a1b36dcb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8bebad80b namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: f0c9f243-8aa9-4315-9f50-0b86083a3e12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8c1683b0d namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: ab487d7e-a45f-4dc7-bb32-4e46a2e51db7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8c1d927ac namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 3c1a0ea8-2d94-4dd6-bf3d-59eaf5f17d7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8c1e3438a namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 414eb888-3952-45b5-a50f-864a139c056a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8c61390b0 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: bd2fc1d1-d307-41b5-a413-a2c8028045e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:46:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:51Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8c6836cbf namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 4688e82f-efbc-4e94-b691-463697d47f1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:47:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:47:28Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62cd8ffab7a7e namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 97399f7e-ceab-4ee8-aa45-e9b31fd69922 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62d599f788ef4 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 04b4e79a-e1ad-4b4c-af3e-a771a0a7c60f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: e2fe5e12-9e83-432a-b26b-4cfd217203a7 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-390a8-predictor-6749bbb468-v5k5t.18d62d599f794dc5 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: d14517e5-b682-4eb6-b7bc-2121b88f2108 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-390a8-predictor-6749bbb468 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 1328c5ea-5e2d-4fbd-a12e-af73e2f64ae9 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: 'Created pod: success-200-isvc-390a8-predictor-6749bbb468-v5k5t' metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:50Z" name: success-200-isvc-390a8-predictor-6749bbb468.18d62cd876ea01d8 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 2dceae14-b889-4b0f-aed1-0f04ddf4b858 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 64757d63-5805-4641-8f9d-4fa32ad385f8 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Scaled up replica set success-200-isvc-390a8-predictor-6749bbb468 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:50Z" name: success-200-isvc-390a8-predictor.18d62cd8751050dc namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: c34ad23b-9b5d-42bb-a08d-df746c974a8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: f4659e82-2ee0-4eaf-a0fc-28cc2ff94e4d kind: Event lastTimestamp: "2026-09-17T17:47:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:20Z" name: success-200-isvc-390a8-predictor.18d62cdbf35ffece namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 9d2396c6-c25e-4cc9-aa7c-9ab4cc59dd22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: f4659e82-2ee0-4eaf-a0fc-28cc2ff94e4d kind: Event lastTimestamp: "2026-09-17T17:47:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:20Z" name: success-200-isvc-390a8-predictor.18d62cdbf3c92aa8 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 49cbca98-b4c8-4a4b-8f1f-3ef4392a8520 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: f4659e82-2ee0-4eaf-a0fc-28cc2ff94e4d kind: Event lastTimestamp: "2026-09-17T17:48:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:20Z" name: success-200-isvc-390a8-predictor.18d62ce2f12953bd namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 1a32e1d3-a6e8-41de-b9ad-c0dcc7ef287b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: f4659e82-2ee0-4eaf-a0fc-28cc2ff94e4d kind: Event lastTimestamp: "2026-09-17T17:48:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:48:20Z" name: success-200-isvc-390a8-predictor.18d62ce2f17a27c2 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 0e3ee39a-6b2c-48e7-8e76-df5b49c95d68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390a8 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: c72facb9-920e-43a0-8841-a0f1e8f34704 kind: Event lastTimestamp: "2026-09-17T17:47:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:38Z" name: success-200-isvc-390a8.18d62cd875155488 namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 0279ecd7-fd1b-4017-9863-44879f829b2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:47:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390a8 namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: c72facb9-920e-43a0-8841-a0f1e8f34704 kind: Event lastTimestamp: "2026-09-17T17:47:38Z" message: InferenceService [success-200-isvc-390a8] is Ready metadata: creationTimestamp: "2026-09-17T17:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:47:38Z" name: success-200-isvc-390a8.18d62ce3bb2820f9 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: ddefabd9-52b6-4935-80e4-ec4696e7a1f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:06:43.860448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee65d7c878 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 08bf463d-9012-4240-9673-f8e4787e147d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee88466d04 namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: f202b45b-fc5e-4b8b-9c40-1e128f055d5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee89b63494 namespace: kserve-ci-e2e-test resourceVersion: "25839" uid: 72cc886c-0a7a-4fc1-866b-748823cae72b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee8d35a589 namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 94cdfeaf-be2b-4ef7-ac69-e970b70daf50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee8dadb1ae namespace: kserve-ci-e2e-test resourceVersion: "25845" uid: 1a9114f9-4c34-4699-a847-a782e099c3ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee8dbb9e5e namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: 33ec979d-f49a-4b4b-9527-1986610f8bf9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee91b155dd namespace: kserve-ci-e2e-test resourceVersion: "25848" uid: 06b3daa6-8960-4bba-9a18-8b9a01621abb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62dee922c03bf namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: b6b0f2ee-da30-4763-827b-ce0935c889f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:07:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:22Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62deee45be6a0 namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: 68bb54a1-7e7d-42b8-bdf3-acafa337c7ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:15:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:15:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:15:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:15:58Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62e6f91e65fcd namespace: kserve-ci-e2e-test resourceVersion: "30314" uid: ae9a7b12-b0b3-4a40-aa18-10e5eca26b9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:15:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 7af4e9c8-bc7e-4cf3-a1b3-c28cfb1a0ff7 kind: Event lastTimestamp: "2026-09-17T18:15:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:15:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:15:58Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv.18d62e6f91e6d5ca namespace: kserve-ci-e2e-test resourceVersion: "30318" uid: fadd8907-8276-4a7f-8a66-3978b3bd44f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3ef6b-predictor-77d7f67c46 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 642dbd59-933e-42c3-8fb9-51e6109b37d6 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'Created pod: success-200-isvc-3ef6b-predictor-77d7f67c46-4dshv' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-3ef6b-predictor-77d7f67c46.18d62dee65643313 namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: bdd52ceb-95c1-4957-ab76-cd3c592c7cfa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 9b54d4fe-f37f-418d-8a1f-56a0f2669451 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Scaled up replica set success-200-isvc-3ef6b-predictor-77d7f67c46 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-3ef6b-predictor.18d62dee632783d5 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: a70d1257-e078-4db3-9e51-89a6b0a9080a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: c1c0f707-f9ab-4c86-89c1-077ef3d94d2b kind: Event lastTimestamp: "2026-09-17T18:07:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:13Z" name: success-200-isvc-3ef6b-predictor.18d62df1e1a9193d namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: ac91fb13-34ef-4da0-9aab-6b05259bc434 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: c1c0f707-f9ab-4c86-89c1-077ef3d94d2b kind: Event lastTimestamp: "2026-09-17T18:07:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:13Z" name: success-200-isvc-3ef6b-predictor.18d62df1e2111360 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 3ccae376-68a9-43e0-b901-b6e1b84469a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: c1c0f707-f9ab-4c86-89c1-077ef3d94d2b kind: Event lastTimestamp: "2026-09-17T18:07:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:58Z" name: success-200-isvc-3ef6b-predictor.18d62df8e096ce50 namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: f366b1c2-4222-4710-bac5-56f644534232 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ef6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: c1c0f707-f9ab-4c86-89c1-077ef3d94d2b kind: Event lastTimestamp: "2026-09-17T18:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:58Z" name: success-200-isvc-3ef6b-predictor.18d62df8e10ef24c namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 293a0198-2372-4dd4-b326-292d77cae3a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 8cbb2fc2-e8a3-483b-8476-6cfeb61b6693 kind: Event lastTimestamp: "2026-09-17T18:07:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:32Z" name: success-200-isvc-3ef6b.18d62dee632b6b7b namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: 950e6be1-2910-4a6f-befa-ee3157d20ec8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 8cbb2fc2-e8a3-483b-8476-6cfeb61b6693 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-3ef6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3ef6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-3ef6b.18d62dee63a34aba namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: 03018ecb-514a-41a4-aabe-13c66891bdf7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 8cbb2fc2-e8a3-483b-8476-6cfeb61b6693 kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3ef6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-3ef6b.18d62dee63a3f05d namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 98aad6b1-869e-4a36-b7d0-6dbb2f4b2fa1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ef6b namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 8cbb2fc2-e8a3-483b-8476-6cfeb61b6693 kind: Event lastTimestamp: "2026-09-17T18:07:32Z" message: InferenceService [success-200-isvc-3ef6b] is Ready metadata: creationTimestamp: "2026-09-17T18:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:32Z" name: success-200-isvc-3ef6b.18d62df9a1c88111 namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: e87c9927-9702-487c-8f73-e39545b90028 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:44:19.489973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:44:19Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb5631e42f9 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: ab7055c8-cfc3-44e1-a8b9-299ad7eee580 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5b8c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:19Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb56fcb4554 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 47a618a2-1d3c-4c24-ad0f-d40c6fef18b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:44:20Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb5a90df8f6 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: ce8f8cae-0ed8-46a5-aadc-1b2956456eef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:20Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" metadata: creationTimestamp: "2026-09-17T17:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:20Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb5aa89a0ea namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: bdc9c900-60d7-4dec-893a-984f31b7e4b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" in 16.841s (16.841s including waiting). Image size: 1356859978 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb9966031bb namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 76025ec2-b0e5-468a-bae1-d13b8a636fcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb99c9ba3c3 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: c5d7b435-efbf-4401-ac54-8d303efbe546 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb99dac0bf9 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 98ed0530-62b8-48c8-a6b8-e21eec25da93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:37Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cb99dba0cab namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: 982850a7-c5aa-4dbe-8086-bb17de0559ea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.75s (2.75s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cba41aa8274 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 85f72aa2-2cd2-4da9-be6d-29f7f28cd8ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cba45b754eb namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 66bf350b-4eb7-4782-b03b-64151143f1dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:44:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:44:40Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cba4690e1db namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 0b3eb701-1689-44d1-934b-afeb05dadfe2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:45:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:17Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cba9837581f namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 7ee1c9b7-517b-4b8f-8ea1-9d7b68562c95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:49Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cca5b188300 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 524186de-bcc8-454d-8a45-d5e4c2ab3f4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: b696b601-e784-4c61-963d-fd05aceb4ac6 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:49Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz.18d62cca5b19a7b0 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: cb603406-7f84-495f-a4bd-59d2badb0aea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5b8c1-predictor-c5f74cf7b namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: 3a7b0ae3-18b0-483d-a7fd-8e153d977ca8 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: 'Created pod: success-200-isvc-5b8c1-predictor-c5f74cf7b-l8qrz' metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: success-200-isvc-5b8c1-predictor-c5f74cf7b.18d62cb5629ef646 namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 511b2753-b93a-4627-aa47-54e0c9a9d995 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 2f3079d5-176b-4b5f-873e-874a57940518 kind: Event lastTimestamp: "2026-09-17T17:44:19Z" message: Scaled up replica set success-200-isvc-5b8c1-predictor-c5f74cf7b from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:19Z" name: success-200-isvc-5b8c1-predictor.18d62cb5609cecb3 namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: bcccadc7-ee46-446b-8c36-8389606e78bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: d4d8556c-c75d-43ab-ace5-37e5a568ae06 kind: Event lastTimestamp: "2026-09-17T17:44:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:49Z" name: success-200-isvc-5b8c1-predictor.18d62cb8df0f0e4c namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: f5c6d8d4-2010-4c9c-b1e0-3822f9a1e224 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: d4d8556c-c75d-43ab-ace5-37e5a568ae06 kind: Event lastTimestamp: "2026-09-17T17:44:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:44:49Z" name: success-200-isvc-5b8c1-predictor.18d62cb8df997974 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: 502f3ea6-b349-43de-8db7-a35c5d6137d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: d4d8556c-c75d-43ab-ace5-37e5a568ae06 kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: success-200-isvc-5b8c1-predictor.18d62cbfdd690f2e namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 53587f61-20e5-4763-a979-2c863bd31224 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b8c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: d4d8556c-c75d-43ab-ace5-37e5a568ae06 kind: Event lastTimestamp: "2026-09-17T17:45:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:34Z" name: success-200-isvc-5b8c1-predictor.18d62cbfdde4ab49 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 6dcae1f9-c483-4097-812a-4c92d9a39738 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T17:44:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 06902606-263b-456f-b292-589374ced7fb kind: Event lastTimestamp: "2026-09-17T17:45:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:45:27Z" name: success-200-isvc-5b8c1.18d62cb560a0d18f namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 0310a607-4932-4691-a853-ac87576fcddf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 06902606-263b-456f-b292-589374ced7fb kind: Event lastTimestamp: "2026-09-17T17:45:27Z" message: InferenceService [success-200-isvc-5b8c1] is Ready metadata: creationTimestamp: "2026-09-17T17:45:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:45:27Z" name: success-200-isvc-5b8c1.18d62cc553c55f16 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 03868208-42c8-4013-abf6-4738c7bdfc63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:56:04.966559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59a4ccb844 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: ac55b447-d52b-442f-a601-5c1afda08e01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6b6c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:05Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59b04a0273 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: b2026f18-6b3e-49a8-84c0-b5249c442396 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59eab475a8 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1c9c4bd4-e366-4056-b44e-674aa6fe66d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59ec1f2a33 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 65ea062e-11de-4956-b199-2727c32b2bc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59eeb2da72 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 854aa658-8627-4ba8-84da-e821fe5a7add reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59ef230def namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: 9d273e18-d7a9-4698-acbc-43904057c536 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59ef2cbf95 namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 377fdc5b-60e5-469c-a3c9-892277787b86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59f1d79abe namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 6ef7223e-1fd4-424d-833f-21a5ccf5fee3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:06Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d59f2444b8b namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 8b2e538d-e98e-4140-a9e8-441ed1433894 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:56:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:52Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d5a1cdbd24e namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 97b26dc9-edb5-4016-be2c-5961ef1b1942 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d6d423b1347 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 21cfb60f-0b70-49aa-b9cf-38918ffce7f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2b989a47-1798-48a2-9568-9cb971a3b4a3 kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm.18d62d6d423cb176 namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 98b9070d-cb7a-4618-b75d-755bcc94110e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6b6c1-predictor-d7d64c7db namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 5e93c775-95a2-4d16-be1a-5e73857c57d8 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'Created pod: success-200-isvc-6b6c1-predictor-d7d64c7db-ntzbm' metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-6b6c1-predictor-d7d64c7db.18d62d59a45df64e namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: f65d1c38-02db-4c24-be04-056a56c524f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 4fc739b8-24f7-438c-8435-5180d7d26e07 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: Scaled up replica set success-200-isvc-6b6c1-predictor-d7d64c7db from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-6b6c1-predictor.18d62d59a2bc5139 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: cff50020-fc32-4419-a6e5-a213e0c854c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 619dc279-e982-43ee-afde-c0575e912efe kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-6b6c1-predictor.18d62d5d2167a4b8 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: d1e9a4fa-47c6-4325-9e95-bdf51dd7339d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 619dc279-e982-43ee-afde-c0575e912efe kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-6b6c1-predictor.18d62d5d21cd7747 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 3a1b260f-e99f-4091-85a2-a0ebcd7cba0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:56:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 619dc279-e982-43ee-afde-c0575e912efe kind: Event lastTimestamp: "2026-09-17T17:57:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:20Z" name: success-200-isvc-6b6c1-predictor.18d62d60a09ba68c namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 9ccf1087-7d10-4fcc-892a-ae0f9989c8b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T17:56:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 619dc279-e982-43ee-afde-c0575e912efe kind: Event lastTimestamp: "2026-09-17T17:57:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:20Z" name: success-200-isvc-6b6c1-predictor.18d62d60a10d1601 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 0ec1e19a-b922-4f94-86e4-56711f52f751 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: b2940bbf-0a7f-405b-84a9-fb906187f988 kind: Event lastTimestamp: "2026-09-17T17:57:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:03Z" name: success-200-isvc-6b6c1.18d62d59a2f0f260 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 23408b9c-d8b6-49d1-b76b-9f2faf66ca1d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: b2940bbf-0a7f-405b-84a9-fb906187f988 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-6b6c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-6b6c1.18d62d59a3720350 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: ce35b9ab-5fb5-471c-ab98-9eb8d8cb47b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: b2940bbf-0a7f-405b-84a9-fb906187f988 kind: Event lastTimestamp: "2026-09-17T17:56:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6b6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:56:04Z" name: success-200-isvc-6b6c1.18d62d59a3724cc0 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 882d03b2-3d88-4027-9f45-26f84af7694b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b6c1 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: b2940bbf-0a7f-405b-84a9-fb906187f988 kind: Event lastTimestamp: "2026-09-17T17:57:03Z" message: InferenceService [success-200-isvc-6b6c1] is Ready metadata: creationTimestamp: "2026-09-17T17:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:03Z" name: success-200-isvc-6b6c1.18d62d672c913e7f namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 163af08f-dd32-4cb7-a53b-813ce2700cef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:05:33.587358Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24961" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b5897-predictor-5dd89c9df6-pglvk to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:05:33Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde093d91b9 namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: fbb69447-2066-41c3-8fcf-cb100525890b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde2b4bb95f namespace: kserve-ci-e2e-test resourceVersion: "25011" uid: ae0e8ff3-d21d-4c4c-9843-6e935fab0738 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde2cb927e7 namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: 7bf0c7c8-ced7-42b6-8baa-fcd2b747fcd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde2f828f06 namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: e2f15fca-5db2-42d5-bc40-9fb482f9761c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde2ff223ee namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: ca2bb89c-5ee3-41bc-92a1-8480d1ea742a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde2ffb7d4d namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: c1cbf75e-7257-48c9-b1cc-6ce283bf59a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde32f32f20 namespace: kserve-ci-e2e-test resourceVersion: "25020" uid: 985f3187-84d1-4617-a425-e9600001d8c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:05:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:34Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde336a4ce8 namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 5e263995-56ba-4405-aa27-92946722fcdb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:06:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:11Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dde82a7ac91 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 17ba4927-ab28-4972-904b-cde1397ad1bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dee60d97923 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: c3dcda74-6967-470c-a2d0-f6cfd77ac2ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:06:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:43Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62dee60da2921 namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 28725a05-eff5-4218-a14c-48dd17d9e028 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 4a1f1981-8b92-46be-942d-e4657b35d2ce kind: Event lastTimestamp: "2026-09-17T18:06:46Z" message: 'Readiness probe failed: Get "https://10.133.0.46:8643/healthz": dial tcp 10.133.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:46Z" name: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk.18d62def0adbbee8 namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: b364583b-931a-4dc8-8be9-6487fcfe8382 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b5897-predictor-5dd89c9df6 namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 97f43a99-e1e5-477f-ad4d-f3290bd537ae kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: 'Created pod: success-200-isvc-b5897-predictor-5dd89c9df6-pglvk' metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:05:33Z" name: success-200-isvc-b5897-predictor-5dd89c9df6.18d62dde08ca495d namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: 48a68c0c-67ea-4b82-890f-8f08fbbc88fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: 60ed654b-3e81-4f19-bdd3-f796683ac86b kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Scaled up replica set success-200-isvc-b5897-predictor-5dd89c9df6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:05:33Z" name: success-200-isvc-b5897-predictor.18d62dde057a0cb9 namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: a64832df-e939-4442-8e5b-025212a40c02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24953" uid: 8dc7765f-f98c-4bcb-9822-08c0e9b73990 kind: Event lastTimestamp: "2026-09-17T18:06:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:03Z" name: success-200-isvc-b5897-predictor.18d62de1840844df namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 826f1ab2-5121-43d4-b31c-61fe61a6b6d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "24953" uid: 8dc7765f-f98c-4bcb-9822-08c0e9b73990 kind: Event lastTimestamp: "2026-09-17T18:06:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:03Z" name: success-200-isvc-b5897-predictor.18d62de1848ae209 namespace: kserve-ci-e2e-test resourceVersion: "25267" uid: d1a42abe-181a-408e-a3e8-c9186d333604 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: 8dc7765f-f98c-4bcb-9822-08c0e9b73990 kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: success-200-isvc-b5897-predictor.18d62de882578854 namespace: kserve-ci-e2e-test resourceVersion: "25517" uid: e4d8b708-4dd5-4672-a2c9-fb8bb2ed86a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T18:06:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5897-predictor namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: 8dc7765f-f98c-4bcb-9822-08c0e9b73990 kind: Event lastTimestamp: "2026-09-17T18:06:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:33Z" name: success-200-isvc-b5897-predictor.18d62de882d58317 namespace: kserve-ci-e2e-test resourceVersion: "25520" uid: 50999f75-30c2-4164-8806-e3c86c1eaa9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5897 namespace: kserve-ci-e2e-test resourceVersion: "24943" uid: 38cef292-2163-415d-930b-ea38cdd4fd58 kind: Event lastTimestamp: "2026-09-17T18:06:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:21Z" name: success-200-isvc-b5897.18d62dde0588c42a namespace: kserve-ci-e2e-test resourceVersion: "25419" uid: 3977d5f7-765e-434b-ba11-f096376425dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5897 namespace: kserve-ci-e2e-test resourceVersion: "25416" uid: 38cef292-2163-415d-930b-ea38cdd4fd58 kind: Event lastTimestamp: "2026-09-17T18:06:21Z" message: InferenceService [success-200-isvc-b5897] is Ready metadata: creationTimestamp: "2026-09-17T18:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:06:21Z" name: success-200-isvc-b5897.18d62de94252566c namespace: kserve-ci-e2e-test resourceVersion: "25417" uid: c309dce1-0021-401b-baed-5c88ad1b37c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:56:18.712151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c5c33-predictor-7d76dcc468-tldgv to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5cd819d22f namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 1d598dc5-4f7e-4587-8217-5c345c40cd60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c5c33-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5ce23dbdfa namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: 80211ea4-590f-4257-ada9-a8bca5324286 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d1e393bc4 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 136f3985-80b3-4ff1-a484-2d1da6937042 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d1f89d32b namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 0b61fb7c-674b-4645-aee0-a17992191d49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d2293fa8d namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 098267df-14c7-44d7-8255-6b01a6c22d1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d2305d0c6 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 57aacf4d-5ed2-4ea0-8d19-01a5fe27baf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:19Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d230f7cd4 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: f163abe6-e8cb-441e-922c-c4fa1e721c0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:20Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d25b1788d namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: b9bf8b28-215a-4e7e-b2b9-3eac34461c35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:20Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d263139ac namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 3fe522a9-d40c-4b27-9dcc-1f6bc9b47340 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T17:56:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:56:57Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62d5d28314f0d namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 9793b654-9677-4bd7-8dc9-98ca2dc4d17a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:33Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62dde00d388ed namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 7d1770eb-ccc1-4d82-8395-26c7b3a619c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2e69c7c5-5af7-477f-92db-cea0c52943b1 kind: Event lastTimestamp: "2026-09-17T18:05:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:05:33Z" name: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv.18d62dde00d46208 namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: f08b823e-3b5f-4289-abbe-fa72df8e9ac1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c5c33-predictor-7d76dcc468 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 3350dda5-5247-424d-9732-3ae06a7e58e4 kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: 'Created pod: success-200-isvc-c5c33-predictor-7d76dcc468-tldgv' metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-c5c33-predictor-7d76dcc468.18d62d5cd7a86634 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 1f2c17bd-e2b3-4f34-980e-f2a3c520dfd5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 609fe7d3-f969-4436-97bd-1cdd1903980b kind: Event lastTimestamp: "2026-09-17T17:56:18Z" message: Scaled up replica set success-200-isvc-c5c33-predictor-7d76dcc468 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:56:18Z" name: success-200-isvc-c5c33-predictor.18d62d5cd5e780ff namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: b85be317-4e18-460c-a915-b07a82a9abbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:56:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 09f5c026-45da-4bc4-8f7b-d95e75f53f91 kind: Event lastTimestamp: "2026-09-17T17:57:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:03Z" name: success-200-isvc-c5c33-predictor.18d62d6054e9f1c6 namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 84cdd3ae-6c15-44ed-a6a5-99b60aa95b59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:56:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 09f5c026-45da-4bc4-8f7b-d95e75f53f91 kind: Event lastTimestamp: "2026-09-17T17:57:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:03Z" name: success-200-isvc-c5c33-predictor.18d62d60552e80a5 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 6caa6dd2-a2bc-4e1a-91dd-c367e398c883 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 09f5c026-45da-4bc4-8f7b-d95e75f53f91 kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:33Z" name: success-200-isvc-c5c33-predictor.18d62d6ad20a832e namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: c3dfa4b1-9d47-4765-86a4-a0357bbbbbf6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c33-predictor namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 09f5c026-45da-4bc4-8f7b-d95e75f53f91 kind: Event lastTimestamp: "2026-09-17T17:57:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:33Z" name: success-200-isvc-c5c33-predictor.18d62d6ad28d0709 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 8d97c62a-cb96-4bd6-8c6d-91f1e11558a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:56:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 37d89b6d-28f2-4d44-8f6f-03c6a99f371b kind: Event lastTimestamp: "2026-09-17T17:57:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:07Z" name: success-200-isvc-c5c33.18d62d5cd5ecc175 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: eac5e617-7d93-4651-86fd-ed0d7d86abe4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5c33 namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 37d89b6d-28f2-4d44-8f6f-03c6a99f371b kind: Event lastTimestamp: "2026-09-17T17:57:07Z" message: InferenceService [success-200-isvc-c5c33] is Ready metadata: creationTimestamp: "2026-09-17T17:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:57:07Z" name: success-200-isvc-c5c33.18d62d682126c5c6 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: a0fd97f5-681e-4bd4-b5c3-6e19db34cc88 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:06:44.290028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25799" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-de145-predictor-7dbfbb5484-b92v8 to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62dee7f729e2f namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: ade10e90-cb1e-4043-8361-6419af4e65f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-de145-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62dee86b750b0 namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: 9ba989e6-bacf-49f2-9306-359871cd0d49 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25810" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Add eth0 [10.133.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deec5978142 namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: b9e977fd-4a53-4722-8f2f-160cad9bce4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deec6fb3653 namespace: kserve-ci-e2e-test resourceVersion: "25876" uid: e4f72b1e-39f0-4836-b0e0-7672b63ebb97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deec9e8809c namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 8186583b-dbf0-4796-b2b2-46e662861e60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deecaa420da namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 5045af3e-4b77-4da6-89a8-20f7d77fef42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deecab3d4ea namespace: kserve-ci-e2e-test resourceVersion: "25879" uid: 64213679-420f-4aec-838c-d71505d48e46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deecdb2e78d namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 53a243cb-cc20-4fbe-9496-5223ab3e5522 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:06:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:45Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deece3750ad namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: fd5c4e62-9bf9-429c-b51c-d902e61d5be1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T18:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:07:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:31Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62deee44b968e namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: aa865324-f92a-4b5a-bf32-b1e4ec9c450e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:24:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:13Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62ee2c6a00ae4 namespace: kserve-ci-e2e-test resourceVersion: "33755" uid: 51f1bbf6-2ff0-493a-913f-b5845b11e064 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 5a89f15e-8368-4d4e-8bfb-be3196e8a6ce kind: Event lastTimestamp: "2026-09-17T18:24:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:13Z" name: success-200-isvc-de145-predictor-7dbfbb5484-b92v8.18d62ee2c6a0796b namespace: kserve-ci-e2e-test resourceVersion: "33758" uid: 77cc8c8a-5a8d-4244-be64-020bd3327650 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-de145-predictor-7dbfbb5484 namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 68ec9849-01e0-40d5-bc1c-19a938acc570 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: 'Created pod: success-200-isvc-de145-predictor-7dbfbb5484-b92v8' metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-de145-predictor-7dbfbb5484.18d62dee7efd427b namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: d0344f69-9057-4ed0-b98b-1e061b911fbf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 02412a04-27c7-4710-82d9-038914e868a2 kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Scaled up replica set success-200-isvc-de145-predictor-7dbfbb5484 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-de145-predictor.18d62dee7d05bc19 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 77b32d54-0714-41c2-b865-638d1a571b5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 0a48b58b-6302-4753-a8d4-b734758f4676 kind: Event lastTimestamp: "2026-09-17T18:06:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:59Z" name: success-200-isvc-de145-predictor.18d62df1fb96be63 namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: e432461c-45a7-49fc-ab1d-e9a31da7e649 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 0a48b58b-6302-4753-a8d4-b734758f4676 kind: Event lastTimestamp: "2026-09-17T18:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:06:59Z" name: success-200-isvc-de145-predictor.18d62df1fc07c4eb namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 0e5e0050-3cf6-4bb3-b719-e2f4258ed97e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 0a48b58b-6302-4753-a8d4-b734758f4676 kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: success-200-isvc-de145-predictor.18d62df57ad4abee namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: eb4f6c50-36b5-4a0f-81fb-830573173af1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T18:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de145-predictor namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 0a48b58b-6302-4753-a8d4-b734758f4676 kind: Event lastTimestamp: "2026-09-17T18:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T18:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:59Z" name: success-200-isvc-de145-predictor.18d62df57b556c33 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: cbc62f05-e638-4694-ae8a-7fc35255fc05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de145 namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 840e8859-b27a-4b38-982e-4f0be50b63ef kind: Event lastTimestamp: "2026-09-17T18:07:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:42Z" name: success-200-isvc-de145.18d62dee7d32e4b9 namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: 7202fe25-ed0f-4322-aa2e-3d5533b768c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de145 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 840e8859-b27a-4b38-982e-4f0be50b63ef kind: Event lastTimestamp: "2026-09-17T18:07:42Z" message: InferenceService [success-200-isvc-de145] is Ready metadata: creationTimestamp: "2026-09-17T18:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:42Z" name: success-200-isvc-de145.18d62dfbf41a60e0 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 488a07e0-b023-4b8d-b0fc-3f635873ac16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:57:29.348421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f13ae-predictor-b68795d4b-lvvqr to ip-10-0-137-20.ec2.internal metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d4a59b2f7 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 46dd90ea-247b-4187-abb9-baaeec4e5131 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f13ae-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d57fbdccc namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: d19f965c-00b6-4d46-a9e6-50ec7d87c7c3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d8ff6d317 namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: 58d97e5d-84dc-4b56-a948-111ec26d5912 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1903" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d914fdc73 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: aa002397-4c0f-487d-9825-aa4091a74709 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d944402ec namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 85bca543-c357-4500-abb4-e47b60667161 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d94b32e1e namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 5dfb01e8-7bac-42be-bfe5-db41a5d64ade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d94bd9ea8 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 263427bd-3058-48e0-849a-0da3f59cbf5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d97907a2f namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 242bbffe-8a90-4951-8758-dbfd6e0265e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T17:57:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T17:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:57:30Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6d9804b33e namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 19e0c95f-cf96-44c7-a1e4-2e0e5858c1a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T17:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T18:06:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T17:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:47Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62d6dc4d68dd7 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: f1d71c6d-6fb3-4cce-b31d-3c3aa46a252a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62dee758b4f40 namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: b3343dcf-581c-4928-bb18-0af368f6ca62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T18:06:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T18:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:44Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62dee758bc8e5 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: bd613d9c-072f-4fa4-ae26-c9e66d055d58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 198a6626-b57d-460d-a29b-a7932f83006a kind: Event lastTimestamp: "2026-09-17T18:06:47Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T18:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:06:47Z" name: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr.18d62def39347d63 namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: 0ce4254c-ea0a-4b10-9334-830aed2e8f37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-20.ec2.internal source: component: kubelet host: ip-10-0-137-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f13ae-predictor-b68795d4b namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 19e8eb83-4fc1-4452-97bb-f5c7273fc5ac kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: 'Created pod: success-200-isvc-f13ae-predictor-b68795d4b-lvvqr' metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-f13ae-predictor-b68795d4b.18d62d6d49a4c197 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: d8b21540-aa65-4798-bd41-f579cc576114 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 6df78f48-321b-47bc-8b91-c267cc302ffa kind: Event lastTimestamp: "2026-09-17T17:57:29Z" message: Scaled up replica set success-200-isvc-f13ae-predictor-b68795d4b from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:57:29Z" name: success-200-isvc-f13ae-predictor.18d62d6d47bdf4ef namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 646a15ac-0e0d-4111-bec8-a8ceba289136 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 5c69c689-d463-4e37-8027-010b6acac837 kind: Event lastTimestamp: "2026-09-17T17:58:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:14Z" name: success-200-isvc-f13ae-predictor.18d62d70c66ab786 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 2952dd2f-9f29-48bc-9c07-40429a23da3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T17:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 5c69c689-d463-4e37-8027-010b6acac837 kind: Event lastTimestamp: "2026-09-17T17:58:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:14Z" name: success-200-isvc-f13ae-predictor.18d62d70c6e8a0a1 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 40bba10c-6adf-4fea-9e51-946f7fb4bea8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 5c69c689-d463-4e37-8027-010b6acac837 kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-f13ae-predictor.18d62d7b43f9d34b namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 25b7631d-f7a9-4500-8f0e-3b0a6b6aa2c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:58:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f13ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 5c69c689-d463-4e37-8027-010b6acac837 kind: Event lastTimestamp: "2026-09-17T17:58:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T17:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:58:44Z" name: success-200-isvc-f13ae-predictor.18d62d7b449c2470 namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 5bcea8a3-5e02-4916-91dd-bae76057d1f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: c423acca-5c79-4d9b-b8ef-02421700f226 kind: Event lastTimestamp: "2026-09-17T17:58:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T17:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:17Z" name: success-200-isvc-f13ae.18d62d6d47cd32b4 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 33589327-00ef-4e6a-8df2-80da87e82485 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:58:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f13ae namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: c423acca-5c79-4d9b-b8ef-02421700f226 kind: Event lastTimestamp: "2026-09-17T17:58:17Z" message: InferenceService [success-200-isvc-f13ae] is Ready metadata: creationTimestamp: "2026-09-17T17:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:58:17Z" name: success-200-isvc-f13ae.18d62d7880073f96 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 72d3124e-abe9-4380-a3d9-f7555cc09bb5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:46:49.700242Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-2d039-5d68f9b84-9fcp8 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:46:49Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62cd85c5911c7 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: f2f45732-495b-4794-9b84-10e6d162a4ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:46:50Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62cd87e8a4411 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 619987cf-56b2-41df-af81-81f61eb4046c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2d039} kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62cd88045d8ed namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: eb9c5d3a-31d5-49d7-a8f7-d97c59773c8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2d039} kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: 'Created container: switch-graph-2d039' metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62cd8832af4ba namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: e6861b47-e5e7-4d25-9594-f4decc3f4cba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2d039} kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:46:50Z" message: Started container switch-graph-2d039 metadata: creationTimestamp: "2026-09-17T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:50Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62cd88398d6c3 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 4eafba63-9ea3-440a-a0ea-e5ec101d3f5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2d039} kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:55:04Z" message: Stopping container switch-graph-2d039 metadata: creationTimestamp: "2026-09-17T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:04Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62d4b7bc87b18 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: cd9f6ca1-663c-481d-bd06-391eecd207e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2d039} kind: Pod name: switch-graph-2d039-5d68f9b84-9fcp8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 58fe8576-f9ed-44ee-85fa-7d1c52a7c157 kind: Event lastTimestamp: "2026-09-17T17:55:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:55:32Z" name: switch-graph-2d039-5d68f9b84-9fcp8.18d62d4c3a484ffa namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 150fcb01-1ea1-453f-a9ff-56c3c1a1010d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-2d039-5d68f9b84 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 1c7f0f42-ea4e-49cd-9aab-82a8f621f724 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: 'Created pod: switch-graph-2d039-5d68f9b84-9fcp8' metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: switch-graph-2d039-5d68f9b84.18d62cd85be1583b namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 91b2fe79-3797-4cf0-bb48-911af0dece69 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 84398bed-9346-4b5e-8e58-23d1f71b22e2 kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: Scaled up replica set switch-graph-2d039-5d68f9b84 from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:46:49Z" name: switch-graph-2d039.18d62cd85afab3ef namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 41a2e60c-2606-4a6b-8b0d-ed753862f64c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: b0feafdd-2a7c-433b-8b1f-ce2847004fed kind: Event lastTimestamp: "2026-09-17T17:46:49Z" message: InferenceGraph [switch-graph-2d039] is Ready metadata: creationTimestamp: "2026-09-17T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:46:49Z" name: switch-graph-2d039.18d62cd86102f803 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: eddbfd42-ee86-4623-871f-89d1fee00fe4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 9827e5b2-e8b5-4689-bb4b-b52d65d3f9b5 kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:19Z" name: switch-graph-2d039.18d62cdbd9d9efb5 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 0bcd984c-2299-499c-ad00-20886c7dee89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T17:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2d039 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 9827e5b2-e8b5-4689-bb4b-b52d65d3f9b5 kind: Event lastTimestamp: "2026-09-17T17:47:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:47:19Z" name: switch-graph-2d039.18d62cdbda467444 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 34b34dff-00dd-4861-9a17-c9fc5f5062c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T17:45:39.227955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5b8c1-7f5b78944d-ngml2 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc7f3df46b9 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 509f724b-4c0e-4d1d-8d68-80a2e0b6597f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:39Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc815aa9db0 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: ec9ba2ae-2df9-4e00-b6a2-f45bcda3872d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc8171527fe namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: ca9af36e-58b7-4361-9ead-3105086820a9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" in 2.323s (2.323s including waiting). Image size: 211747396 bytes.' metadata: creationTimestamp: "2026-09-17T17:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:42Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc8a18c1b2a namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 864f969d-5431-48c8-881f-793c8ddfdbeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:42Z" message: 'Created container: switch-graph-5b8c1' metadata: creationTimestamp: "2026-09-17T17:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:42Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc8a499982e namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: d03282c6-fab2-4726-b10f-09260a24468d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:42Z" message: Started container switch-graph-5b8c1 metadata: creationTimestamp: "2026-09-17T17:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:42Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cc8a50581bf namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 62f17384-228d-4fdf-b9c1-eb92a06640f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:45:49Z" message: Stopping container switch-graph-5b8c1 metadata: creationTimestamp: "2026-09-17T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:45:49Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62cca54e4d574 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: cc520fff-ff39-4bbb-807b-789f7836dee5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T17:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5b8c1} kind: Pod name: switch-graph-5b8c1-7f5b78944d-ngml2 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7f5664c0-32ef-4d2a-a242-53ecdb80cfa3 kind: Event lastTimestamp: "2026-09-17T17:46:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T17:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T17:46:19Z" name: switch-graph-5b8c1-7f5b78944d-ngml2.18d62ccb6943a30a namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: ea6f828b-7226-4706-9991-d50122851a95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5b8c1-7f5b78944d namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: c9497ec5-213e-4a81-be63-ce6b86f38302 kind: Event lastTimestamp: "2026-09-17T17:45:39Z" message: 'Created pod: switch-graph-5b8c1-7f5b78944d-ngml2' metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1-7f5b78944d.18d62cc7f356e20f namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 68e60512-a0e3-4632-80b2-7b96fcce29f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: f1447d9b-a658-4b23-943d-ba6464a5f5c9 kind: Event lastTimestamp: "2026-09-17T17:45:39Z" message: Scaled up replica set switch-graph-5b8c1-7f5b78944d from 0 to 1 metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1.18d62cc7f1d022a3 namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: d526693e-5fc4-4da5-83c5-abe47900498d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T17:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5b8c1 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: 87414386-f1f7-4215-aabd-b41491f1057b kind: Event lastTimestamp: "2026-09-17T17:45:39Z" message: InferenceGraph [switch-graph-5b8c1] is Ready metadata: creationTimestamp: "2026-09-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T17:45:39Z" name: switch-graph-5b8c1.18d62cc7f9c84b51 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 465cac69-8251-40fc-bdd4-55fbc7c3242f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T18:07:54.206621Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-de145-59f9568d97-5nvf7 to ip-10-0-128-248.ec2.internal metadata: creationTimestamp: "2026-09-17T18:07:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T18:07:54Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dfec6cd2c8a namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: cb308176-b8b6-44b6-94df-9f323b8bcf51 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c98c6d485-dmg49 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:07:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-de145-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:54Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dfed27002ff namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 46369de5-7ccd-44dd-8231-801047eaea9a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:07:55Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T18:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T18:07:55Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dff0d086142 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: cbacebce-b58c-4fd2-ae48-acc219154e1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-de145} kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:07:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3663a14fbe8f475ebbdcabf8d9547595eec55855d8414670c1d7c06da85f80f1" already present on machine metadata: creationTimestamp: "2026-09-17T18:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:55Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dff0ea64a46 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: ce35bd95-8429-4348-9df9-ff9e0f22b3ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-de145} kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:07:55Z" message: 'Created container: switch-graph-de145' metadata: creationTimestamp: "2026-09-17T18:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:55Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dff119df30a namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 435a512e-3d3e-4898-869f-667b15e53bcf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-de145} kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:07:55Z" message: Started container switch-graph-de145 metadata: creationTimestamp: "2026-09-17T18:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:07:55Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62dff12227f1e namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 4a7cec4b-3660-45cf-852a-f93ceb137f54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-de145} kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:24:13Z" message: Stopping container switch-graph-de145 metadata: creationTimestamp: "2026-09-17T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:13Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62ee2bf6abe83 namespace: kserve-ci-e2e-test resourceVersion: "33733" uid: b6422197-2e27-4d2d-94d7-74cd3ca067ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T18:24:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-de145} kind: Pod name: switch-graph-de145-59f9568d97-5nvf7 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: cc8af9b7-3d17-4f23-976e-606bdd1d6890 kind: Event lastTimestamp: "2026-09-17T18:24:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T18:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T18:24:37Z" name: switch-graph-de145-59f9568d97-5nvf7.18d62ee3b1414bb5 namespace: kserve-ci-e2e-test resourceVersion: "33980" uid: 089b9d04-e6f8-48e7-9f46-9c13c7deda4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-248.ec2.internal source: component: kubelet host: ip-10-0-128-248.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-de145-59f9568d97 namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: e6080098-f3be-4661-a27e-4ab575ab792b kind: Event lastTimestamp: "2026-09-17T18:07:54Z" message: 'Created pod: switch-graph-de145-59f9568d97-5nvf7' metadata: creationTimestamp: "2026-09-17T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:54Z" name: switch-graph-de145-59f9568d97.18d62dfec654c52a namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 46dd67bc-e72e-4249-86f3-bd7f0971d3b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-de145 namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: 8d090da3-026b-4c3c-a8b6-e4b40c7c3e76 kind: Event lastTimestamp: "2026-09-17T18:07:54Z" message: Scaled up replica set switch-graph-de145-59f9568d97 from 0 to 1 metadata: creationTimestamp: "2026-09-17T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:07:54Z" name: switch-graph-de145.18d62dfec56d03c9 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 61ebb177-2e91-4056-a93f-3a11cce8efe7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T18:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-de145 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 6252afb3-c057-48b1-9987-96d3092c9b1e kind: Event lastTimestamp: "2026-09-17T18:07:54Z" message: InferenceGraph [switch-graph-de145] is Ready metadata: creationTimestamp: "2026-09-17T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T18:07:54Z" name: switch-graph-de145.18d62dfecba8d22a namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: 9e6f0e7e-78f7-4337-acc9-c70906ff721e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-de145 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: eeecfcc6-f11c-4422-9b8d-f97711f63978 kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: switch-graph-de145.18d62e02446704f3 namespace: kserve-ci-e2e-test resourceVersion: "27215" uid: fe16fd7f-8687-43b9-9176-cb4c6ed7c85b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T18:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-de145 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: eeecfcc6-f11c-4422-9b8d-f97711f63978 kind: Event lastTimestamp: "2026-09-17T18:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T18:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T18:08:39Z" name: switch-graph-de145.18d62e0244a64cc4 namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: 769d7092-6e60-4f24-aaf0-f3372c98bd5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34007"