--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:09.174673Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-25c73-765966cfb4-d8pwg to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be41fb17a68698 namespace: kserve-ci-e2e-test resourceVersion: "21689" uid: 46f7bb74-ef09-4a00-994d-33533da284ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be41fb31303070 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 77b7b09b-b6eb-41d9-bded-a5ac2c4960bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-25c73} kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be41fb33145d1d namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 4ae62e95-bec4-4da7-aaf8-b4f0afc3723b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-25c73} kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: 'Created container: ensemble-graph-25c73' metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be41fb357e86a7 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 29bbf14c-ad65-4f4f-9363-060877f8993c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-25c73} kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: Started container ensemble-graph-25c73 metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be41fb35eddc3b namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 8f49ef99-94b6-4463-8a4c-6cee23bd503a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-25c73} kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:52:23Z" message: Stopping container ensemble-graph-25c73 metadata: creationTimestamp: "2026-07-01T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:23Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be426e42d1d23d namespace: kserve-ci-e2e-test resourceVersion: "25189" uid: e7c4206d-b465-4160-b729-4a740427c5a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-25c73} kind: Pod name: ensemble-graph-25c73-765966cfb4-d8pwg namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 3a34d86f-d248-48de-b206-3a031a627764 kind: Event lastTimestamp: "2026-07-01T19:52:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:50Z" name: ensemble-graph-25c73-765966cfb4-d8pwg.18be426eb79422ef namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: f27b7e68-7877-44e4-87e4-abdb4da87510 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-25c73-765966cfb4 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 40901601-8b84-426f-8074-475c164c76ff kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: 'Created pod: ensemble-graph-25c73-765966cfb4-d8pwg' metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73-765966cfb4.18be41fb172a7c90 namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 06c5e35d-cd37-4908-9fae-01690309380b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: e28b38c8-a5f9-49ba-a50c-e21177c0c8c6 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: Scaled up replica set ensemble-graph-25c73-765966cfb4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73.18be41fb160d331a namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 013f997b-bb23-45f0-b4e1-a6a2829db79d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: 45185c05-a026-476b-94c0-173bc1e0ce42 kind: Event lastTimestamp: "2026-07-01T19:44:09Z" message: InferenceGraph [ensemble-graph-25c73] is Ready metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:09Z" name: ensemble-graph-25c73.18be41fb1bcdc546 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 508ffdc0-f1db-4bca-b13c-c572b7e8c39a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 218abe8e-aaa4-4b18-8254-d1091edbacbd kind: Event lastTimestamp: "2026-07-01T19:44:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:24Z" name: ensemble-graph-25c73.18be41fe94931cfb namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 4f8c6332-5bd5-4260-b68a-2553cba3bbeb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 218abe8e-aaa4-4b18-8254-d1091edbacbd kind: Event lastTimestamp: "2026-07-01T19:44:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:24Z" name: ensemble-graph-25c73.18be41fe9503ee51 namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9a2adde6-36de-44f6-b43b-bda8559a4b8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 218abe8e-aaa4-4b18-8254-d1091edbacbd kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: ensemble-graph-25c73.18be42021456693d namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: f55a889e-f149-4ced-a936-e5ee39a46e79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 218abe8e-aaa4-4b18-8254-d1091edbacbd kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: ensemble-graph-25c73.18be420214d9f18e namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: fbc03fbe-c636-48b1-8d09-cbb530ebc1fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:48.849210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-44b6c-867f7d49d8-w2qst to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:48Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41e863e147bf namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: 6bb15390-a088-4020-8434-a95903556666 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:49Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:42:49Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:49Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41e87d112187 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: fa706a2d-a321-4edb-ae3f-c09243b0e186 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-44b6c} kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:42:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:49Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41e87e583c30 namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 06f16245-4a00-4dc6-977a-69124c3f4bbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-44b6c} kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:42:49Z" message: 'Created container: ensemble-graph-44b6c' metadata: creationTimestamp: "2026-07-01T19:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:49Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41e881083521 namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: 2ceda786-47a7-4907-ac36-2588aa65710e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-44b6c} kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:42:49Z" message: Started container ensemble-graph-44b6c metadata: creationTimestamp: "2026-07-01T19:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:49Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41e881789956 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 7c9fda46-6ca4-4999-acbc-38305e991b26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-44b6c} kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:42:58Z" message: Stopping container ensemble-graph-44b6c metadata: creationTimestamp: "2026-07-01T19:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:58Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41eabc58968d namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: b604dbbd-2959-4d9b-aec2-6b7d54e7fc47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-44b6c} kind: Pod name: ensemble-graph-44b6c-867f7d49d8-w2qst namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: a7fe6464-7430-43d8-9b79-b5933d2cd88b kind: Event lastTimestamp: "2026-07-01T19:43:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:25Z" name: ensemble-graph-44b6c-867f7d49d8-w2qst.18be41eb19d2ecdd namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: adaf2ffd-12af-40b9-abbe-3fc7b0d82ca6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-44b6c-867f7d49d8 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: c307f204-978f-4d85-8714-14535b941946 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: 'Created pod: ensemble-graph-44b6c-867f7d49d8-w2qst' metadata: creationTimestamp: "2026-07-01T19:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: ensemble-graph-44b6c-867f7d49d8.18be41e8636c5082 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 96c33057-5fd6-41c3-98c6-c410b302970c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-44b6c namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 31e08d66-0ddd-4cd3-becb-84a2981abda5 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: Scaled up replica set ensemble-graph-44b6c-867f7d49d8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: ensemble-graph-44b6c.18be41e862c73d3e namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 747ec814-4bdc-474f-88c0-75573ca49003 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-44b6c namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: d695e9ea-2ad0-4d5e-bdaa-12929f70da84 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: InferenceGraph [ensemble-graph-44b6c] is Ready metadata: creationTimestamp: "2026-07-01T19:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:48Z" name: ensemble-graph-44b6c.18be41e868172f70 namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 4c725caa-0899-4503-ae46-298946dda0e6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:54.138104Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-00068-predictor-589d75fd87-jdckw to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be427551e8cff4 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: e6ac9031-6557-497d-8b6f-3bbd7c77bf13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-00068-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be42755f54346d namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 61c15ad3-d882-4678-87db-8674b17f090b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: Add eth0 [10.132.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be4275901d1e07 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: a4fc6242-bc7c-4463-b4a7-77bab5b8d877 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be4275917d8403 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 0503e8e9-f419-4811-8130-c3e62bfb9472 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be427595600f41 namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: d41c010c-251d-478c-82df-77757d45565e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be427595d245f2 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 16056af3-2fce-4fed-b564-e53ed8e9ccf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be427595dbd39c namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: d66a812f-5bfc-4aab-95ba-ca0a30c1648b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be427598a93dea namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: af90652b-de4b-4bf5-b41d-ac2bb9bc5040 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:52:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:55Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be42759915efac namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: a31d0255-40ec-40ac-aa8d-3c4f7f029080 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T19:53:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:32Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be4275bdb84ba8 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 60587500-6ecb-49c0-98e5-9832b0dc3590 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T20:10:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:13Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be43674e9ecd3a namespace: kserve-ci-e2e-test resourceVersion: "33636" uid: 2602d1a8-1617-4d6e-8fb9-774bebafd287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-00068-predictor-589d75fd87-jdckw namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 036888f0-9d4c-4c8f-8d5b-8a9937c823b8 kind: Event lastTimestamp: "2026-07-01T20:10:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:13Z" name: error-404-isvc-00068-predictor-589d75fd87-jdckw.18be43674e9f4ce1 namespace: kserve-ci-e2e-test resourceVersion: "33637" uid: 4e15436a-3b45-402e-a108-89649da3bba9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-00068-predictor-589d75fd87 namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 478727b2-3a12-4a4a-9821-ad981b721b10 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'Created pod: error-404-isvc-00068-predictor-589d75fd87-jdckw' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068-predictor-589d75fd87.18be427551807b19 namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 1e04d876-0cb2-4a8e-af7e-650a8ed7142f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: b15b9cb0-b97e-40c0-aa80-90aa772e9bfb kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Scaled up replica set error-404-isvc-00068-predictor-589d75fd87 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068-predictor.18be42754fc40b54 namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: e617d3c5-a767-4d2f-949c-8f3d5eca0baa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 0fb6dcbd-71c4-4be6-8239-e220047eb0e5 kind: Event lastTimestamp: "2026-07-01T19:53:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:39Z" name: error-404-isvc-00068-predictor.18be4278ce7179ec namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 657f5314-6c9c-4044-a588-7212bd106039 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 0fb6dcbd-71c4-4be6-8239-e220047eb0e5 kind: Event lastTimestamp: "2026-07-01T19:53:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:39Z" name: error-404-isvc-00068-predictor.18be4278ced8ead5 namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 60887013-8b51-4288-93e3-9bde60dcd6a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 0fb6dcbd-71c4-4be6-8239-e220047eb0e5 kind: Event lastTimestamp: "2026-07-01T19:54:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:09Z" name: error-404-isvc-00068-predictor.18be427c4d80db7d namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 98ef3399-9e11-41c6-aba7-fda86f8ce2ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 0fb6dcbd-71c4-4be6-8239-e220047eb0e5 kind: Event lastTimestamp: "2026-07-01T19:54:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:09Z" name: error-404-isvc-00068-predictor.18be427c4de29a4a namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: ed77c737-4d13-4655-8959-34cd924bd75a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: ea54a779-f6cf-4638-9504-deed7201e470 kind: Event lastTimestamp: "2026-07-01T19:53:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:43Z" name: error-404-isvc-00068.18be42754fee01a8 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 41026ec8-0498-40dc-bf25-7549724f59e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: ea54a779-f6cf-4638-9504-deed7201e470 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-00068": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-00068": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068.18be4275506123f7 namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 0501ad2a-dfb7-4a26-893d-c9373e3c1cd5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: ea54a779-f6cf-4638-9504-deed7201e470 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-00068": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:54Z" name: error-404-isvc-00068.18be427550616769 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: bfb7827f-feed-47c1-887a-8841ec64e40e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: ea54a779-f6cf-4638-9504-deed7201e470 kind: Event lastTimestamp: "2026-07-01T19:53:43Z" message: InferenceService [error-404-isvc-00068] is Ready metadata: creationTimestamp: "2026-07-01T19:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:43Z" name: error-404-isvc-00068.18be4280b548bcdf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: 5ab27357-c2eb-4e32-b78c-7fe1a42cbdd1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:59.189623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-25c73-predictor-6ddf4c5487-md7nf to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eacc375f67 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 55945cf2-c538-4edb-ab45-21d289c29e27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eae6553cdc namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 900f04dd-af08-4727-933a-65ca0f32f4f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eae7ca3cc6 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 77a4e690-3a1e-41c2-8780-86867a5da3ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eaeb2c35dd namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 28045dd7-b420-4be2-bb8c-9ae8ab4477e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eaebba5f44 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 16473a2a-1869-4072-ae7f-fb5be030dbb4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eaebc31f30 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: ed78de15-b641-42f6-8ad5-f1e4d21469a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eaeeb3499b namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 09350d4b-ff1f-4768-8399-8921c0bd878f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eaef2357a8 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 271238d9-7dc7-44cd-91d2-31e7fd1d12d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:52:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:27Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be41eb3c48e9f3 namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: d6c4a05b-f9b7-4d5e-b5d9-c8347205669e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be426e5948e71e namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 66dc7407-296f-4e70-8c27-9fa3bafebc79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be426e59497362 namespace: kserve-ci-e2e-test resourceVersion: "25283" uid: 52209988-47d9-4a89-85ec-f3c422115685 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: f97ab1d3-1f52-4493-87ec-fcc966e68abc kind: Event lastTimestamp: "2026-07-01T19:52:27Z" message: 'Readiness probe failed: Get "https://10.132.0.47:8643/healthz": dial tcp 10.132.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:27Z" name: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf.18be426f04ef5c94 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 8672a88c-4c38-4ad0-bd8c-e244147ab85c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-25c73-predictor-6ddf4c5487 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 6a8a2454-2dc1-49c7-9bca-85c1d8e46a69 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Created pod: error-404-isvc-25c73-predictor-6ddf4c5487-md7nf' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor-6ddf4c5487.18be41eacbc98ad0 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 39b26a91-a682-41ae-8d33-1715bfc495a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 87f29794-bd0d-4cc3-908a-4dc8dc29dba7 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Scaled up replica set error-404-isvc-25c73-predictor-6ddf4c5487 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73-predictor.18be41eaca312cde namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 8d82c8e1-1687-404f-a488-f2d880afaa3b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: dd69e2c6-5206-48b6-990e-02ae20d39a48 kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:14Z" name: error-404-isvc-25c73-predictor.18be41ee48fd011a namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 34645fa6-4d9d-44d9-82cd-229e30b6bab0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: dd69e2c6-5206-48b6-990e-02ae20d39a48 kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:14Z" name: error-404-isvc-25c73-predictor.18be41ee49920751 namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: b108fd96-de15-4a9c-b11c-2104ebf8f68f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: dd69e2c6-5206-48b6-990e-02ae20d39a48 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:29Z" name: error-404-isvc-25c73-predictor.18be41f1c8513687 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 613d260a-fcee-463b-9b9e-938bc610885a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: dd69e2c6-5206-48b6-990e-02ae20d39a48 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:29Z" name: error-404-isvc-25c73-predictor.18be41f1c8c45182 namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 96a27a8d-ecd8-4a8b-bd9e-8b54079dc4ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: c950aae4-1549-4ada-a862-e7712e96ecca kind: Event lastTimestamp: "2026-07-01T19:43:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:47Z" name: error-404-isvc-25c73.18be41eaca416429 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 0feeaed4-c470-40a6-a41b-d4415551e7e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: c950aae4-1549-4ada-a862-e7712e96ecca kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-25c73": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-25c73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73.18be41eacac235b2 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 8635cd91-4bf7-48bd-b9c4-263c8abf31f6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: c950aae4-1549-4ada-a862-e7712e96ecca kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-25c73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-25c73.18be41eacac27c85 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 245621c5-ccf4-426a-a9b4-c4a13f0ad1c9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: c950aae4-1549-4ada-a862-e7712e96ecca kind: Event lastTimestamp: "2026-07-01T19:43:47Z" message: InferenceService [error-404-isvc-25c73] is Ready metadata: creationTimestamp: "2026-07-01T19:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:47Z" name: error-404-isvc-25c73.18be41f5f8bfd952 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 22b9b7ef-1f12-4262-bff7-ff239725278c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:41:48.949968Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:41:48Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da719b4d42 namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: c7606a97-72e5-490c-8813-aa9fd51e439e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da8b92645d namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7360e3c7-78ce-4ab7-b006-a1577a542d88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da8cd8803c namespace: kserve-ci-e2e-test resourceVersion: "19970" uid: ff3d0561-7976-4ec1-9e97-ccca18307f19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da9015e97f namespace: kserve-ci-e2e-test resourceVersion: "19972" uid: 460e1c18-208a-4fd6-abca-a84ceb72e460 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da90a2cf64 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 0ee0b814-efd8-4655-8eec-82c322e02dc9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da90acd30b namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 8b14efb9-2217-41ee-9528-4917beea2a46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da93cd56c0 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: 84991e63-2d56-4bd2-83bd-24e862b399ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da943c7834 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 9cd1258e-fa9c-4d4a-9307-b309341f2d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:42:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:25Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41da9d29d326 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 98db5e6d-7b67-4df2-b398-0265e4126e7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41eacf064bd3 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 3334e6d2-4db8-4659-bb35-c20c6a8892f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41eacf09d739 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 0848f2c8-48bb-48b6-9f80-e3c4ee84077d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c3ca7119-e603-4b2d-93f8-2eea92ef0800 kind: Event lastTimestamp: "2026-07-01T19:43:00Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:00Z" name: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj.18be41eb25a72d9b namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 752bb013-53d6-406b-8533-c88ccc0ec376 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-44b6c-predictor-65b6d49b65 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: c2bf2de8-bc19-4123-97a3-e3b89cdb6f60 kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: 'Created pod: error-404-isvc-44b6c-predictor-65b6d49b65-pmtmj' metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:41:48Z" name: error-404-isvc-44b6c-predictor-65b6d49b65.18be41da71211ee8 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 2689d36b-49da-4bc3-b8b0-3fa4c9c3eb5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: eeeb6481-8a7c-48d5-bfdb-14ad5328ab41 kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: Scaled up replica set error-404-isvc-44b6c-predictor-65b6d49b65 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:41:48Z" name: error-404-isvc-44b6c-predictor.18be41da6f8acaa9 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: a60cf218-d5dc-468b-a0e9-831d4531a93f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 40a196bb-bff2-4d36-b401-4f4c6ebfe5c3 kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:18Z" name: error-404-isvc-44b6c-predictor.18be41ddee01d609 namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 05ff5aa4-a96c-4e46-9ab2-683bb72c6913 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 40a196bb-bff2-4d36-b401-4f4c6ebfe5c3 kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:18Z" name: error-404-isvc-44b6c-predictor.18be41ddee67d881 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: 74c5ace0-4528-4da5-a10c-56a7b950d92c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 40a196bb-bff2-4d36-b401-4f4c6ebfe5c3 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: error-404-isvc-44b6c-predictor.18be41e4ebba6976 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 898837a8-b557-46f7-a3da-d8bcc9b12ec6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 40a196bb-bff2-4d36-b401-4f4c6ebfe5c3 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: error-404-isvc-44b6c-predictor.18be41e4ec1fa631 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 254c19be-0d6b-4039-934d-5482cdbf04eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b6c namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: bb7562e1-60c5-4acb-9040-3d0570d7abe8 kind: Event lastTimestamp: "2026-07-01T19:42:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:35Z" name: error-404-isvc-44b6c.18be41da6f89a74a namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: e78b9ca6-bbb1-4272-b3c3-bd3e470898a3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b6c namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: bb7562e1-60c5-4acb-9040-3d0570d7abe8 kind: Event lastTimestamp: "2026-07-01T19:42:35Z" message: InferenceService [error-404-isvc-44b6c] is Ready metadata: creationTimestamp: "2026-07-01T19:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:35Z" name: error-404-isvc-44b6c.18be41e5584dfe52 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 50081965-7dd0-44be-afb7-6a266d6bc232 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:34.382168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427eb0a4a35a namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: 5def5095-143f-4e83-bf60-6a88200b7913 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5e51f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ebe5339c3 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 9fe3c95d-0f4a-44cc-a747-a3c20b40805f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: Add eth0 [10.132.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427eee96de3c namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: d9d5d571-0d93-4581-bae4-ed9fa77938d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427eeff144cb namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: faa9b05e-ec75-4376-b205-f4fed67bb609 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ef31578d2 namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: dc563379-34fc-4bbb-b17c-4b6bce0a099c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ef38401ed namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: a4a0a05a-e178-42a1-878a-b3031b9a12aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ef38d27ef namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: ef6f7f59-b9f9-4ee6-9b1f-1619000e5591 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ef732a9e8 namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 7777df5d-a760-4013-8126-7ddf6a1e2e99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427ef7a24a9e namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 300965d9-d306-42e5-b902-97f99d3fd832 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:53:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T20:02:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.55:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:52Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be427f1b1c016f namespace: kserve-ci-e2e-test resourceVersion: "30593" uid: 32ad9a15-7887-44fe-b468-29b47a84a3cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T20:02:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:49Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be42ffe299bb1d namespace: kserve-ci-e2e-test resourceVersion: "30560" uid: 2a9ad9c2-b6f5-446b-abb8-3b47dc5c2e8c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T20:02:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:49Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be42ffe29a34f8 namespace: kserve-ci-e2e-test resourceVersion: "30561" uid: 885fdbee-b8a4-4e9f-9207-85ee53ea8e59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 4cc80bb1-12af-4722-b761-c7cf48940444 kind: Event lastTimestamp: "2026-07-01T20:02:52Z" message: 'Readiness probe failed: Get "https://10.132.0.55:8643/healthz": dial tcp 10.132.0.55:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:52Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh.18be43008f8d36df namespace: kserve-ci-e2e-test resourceVersion: "30592" uid: 617ddf2c-25ea-4ba0-b3c9-acbe9c6b2c06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5e51f-predictor-ccbbff6dc namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 796e970e-5cae-4bb4-9ee1-0801d36d291e kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Created pod: error-404-isvc-5e51f-predictor-ccbbff6dc-42zwh' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f-predictor-ccbbff6dc.18be427eb03360f8 namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: a2d115f9-138c-4c14-ab1a-c4198bcb00af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 73a55adc-1227-4da2-807b-a3ce49edda6c kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Scaled up replica set error-404-isvc-5e51f-predictor-ccbbff6dc from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f-predictor.18be427eaeab38de namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: bf69d641-b1f8-4222-873a-6996ebdb1da5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: e739c3ef-bd86-482d-a536-482bb0834c7a kind: Event lastTimestamp: "2026-07-01T19:53:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:49Z" name: error-404-isvc-5e51f-predictor.18be42822d286749 namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: 571f16f3-f77d-4ce0-a3d9-9e11a3eeda9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: e739c3ef-bd86-482d-a536-482bb0834c7a kind: Event lastTimestamp: "2026-07-01T19:53:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:49Z" name: error-404-isvc-5e51f-predictor.18be42822d982831 namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: 5cb06379-7393-4466-b8d3-4e44b71712fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:54:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: e739c3ef-bd86-482d-a536-482bb0834c7a kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: error-404-isvc-5e51f-predictor.18be4285ac480773 namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: f179b39c-e259-4b0c-be21-b15512ee554c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:54:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: e739c3ef-bd86-482d-a536-482bb0834c7a kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: error-404-isvc-5e51f-predictor.18be4285acbe4aa7 namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: e8ab28ec-e4d6-459d-a275-75814da5523c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: c9152f91-0ace-42c8-a7e8-d5b5e8cb20f5 kind: Event lastTimestamp: "2026-07-01T19:54:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:22Z" name: error-404-isvc-5e51f.18be427eaea943f3 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: c727df48-eba2-4600-a2c3-1926a5576a72 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: c9152f91-0ace-42c8-a7e8-d5b5e8cb20f5 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-5e51f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5e51f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f.18be427eaf0f4d07 namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: 8c181cea-2da0-4244-9630-390cb133c613 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: c9152f91-0ace-42c8-a7e8-d5b5e8cb20f5 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5e51f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-5e51f.18be427eaf0fa5e7 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: bd1be151-fd6d-4fda-ba56-d3b4d63b7323 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: c9152f91-0ace-42c8-a7e8-d5b5e8cb20f5 kind: Event lastTimestamp: "2026-07-01T19:54:22Z" message: InferenceService [error-404-isvc-5e51f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:22Z" name: error-404-isvc-5e51f.18be4289d5f23332 namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: 18e92e7d-b5d4-434d-aa50-d619f18d9040 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:32:33.977093Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41593aa54d89 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: af569bbf-0524-49a5-8ef7-0a2d1dc600f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be415954455d75 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: ee0a5b3e-7a9d-466b-a1c8-7e0012125f0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be4159558d5d27 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: af33c298-a932-4d48-9a70-17d1428cca0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be4159590bc111 namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 0a0b7430-e40d-48d8-a662-43f208a4607d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be4159597c9407 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 5affd60c-84db-42ef-afa6-06cd786b8c36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be415959860b9c namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: aaebeeeb-d66a-4a8f-9771-d23290fa30ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41595ce88a5c namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: e62c9597-45b0-47b5-9234-8139499ca196 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41595d765da4 namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 9069217b-fe38-4d62-8343-d0abac9cd80a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:41:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:52Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be4159b35785bb namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 10f580c9-0b9e-4c21-be2f-eb574a5749d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41da7596459d namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: c0e917b3-53af-47d0-92a8-00d28fa7dc56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41da7596c4f3 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: b342cfb4-95ae-4876-9ce4-ce53f50fc180 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c6ad769a-93dd-4cf1-b3c9-536391a9732c kind: Event lastTimestamp: "2026-07-01T19:41:52Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:52Z" name: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8.18be41db27c01dbe namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: 602a544f-7031-4d95-9267-f939422d191d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6c6c1-predictor-5cd754887c namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: c448ce8c-c10f-4b6f-bec8-66c725e66257 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: 'Created pod: error-404-isvc-6c6c1-predictor-5cd754887c-fnxm8' metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-6c6c1-predictor-5cd754887c.18be41593a03ab61 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: cd9cb21d-ede9-419f-9395-e7f2bf378387 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: d0770b1a-0678-43a1-92b9-c43ca2949bd6 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Scaled up replica set error-404-isvc-6c6c1-predictor-5cd754887c from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-6c6c1-predictor.18be415938602318 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: cd1b6298-fb90-44b4-9051-a76ba019c561 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 34817402-4483-434d-bbaa-622c357ee3f2 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: error-404-isvc-6c6c1-predictor.18be415cb6df1a4a namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: e8682cfa-54c2-40a9-8322-7802614ceacb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 34817402-4483-434d-bbaa-622c357ee3f2 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: error-404-isvc-6c6c1-predictor.18be415cb74135cc namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: bd84bd61-bc35-4d8c-923b-15b5b1b10c01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 34817402-4483-434d-bbaa-622c357ee3f2 kind: Event lastTimestamp: "2026-07-01T19:33:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:49Z" name: error-404-isvc-6c6c1-predictor.18be4163b4a2d36f namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 63a70b0c-beed-4356-a3d8-061f6d630636 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 34817402-4483-434d-bbaa-622c357ee3f2 kind: Event lastTimestamp: "2026-07-01T19:33:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:49Z" name: error-404-isvc-6c6c1-predictor.18be4163b4fa5c43 namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 30840c0b-cc69-491c-81f8-5f512f2b12dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 1c2cbc19-931b-4caa-8663-39aa74a34e39 kind: Event lastTimestamp: "2026-07-01T19:33:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:22Z" name: error-404-isvc-6c6c1.18be415938613cc3 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 34a742ec-dd30-4529-960b-af705f7bbebd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 1c2cbc19-931b-4caa-8663-39aa74a34e39 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-6c6c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-6c6c1.18be415938f32eb6 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: d9b9f196-6fca-4b12-a59b-51b6a7e56afc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 1c2cbc19-931b-4caa-8663-39aa74a34e39 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c6c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-6c6c1.18be415938f374f5 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 132ca762-5f62-4e87-94d0-857d9ca7f424 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 1c2cbc19-931b-4caa-8663-39aa74a34e39 kind: Event lastTimestamp: "2026-07-01T19:33:22Z" message: InferenceService [error-404-isvc-6c6c1] is Ready metadata: creationTimestamp: "2026-07-01T19:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:22Z" name: error-404-isvc-6c6c1.18be416470c2fa0c namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0ca3267d-9223-43e6-bb47-9dffe5ea4669 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:04.062995Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-79b53-predictor-58774dd6b7-tzcmk to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:04Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41444b5906b5 namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: 6672a451-8657-4d00-8f0e-72cdd8fb0bde reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:04Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4144654dfcde namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 1804108d-d621-4e0c-97bb-458e4eb7b395 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41446695dd40 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 72b713a7-532c-450f-af08-fc0050e925ff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" in 15.887s (15.887s including waiting). Image size: 1334626923 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4148199351fd namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 258a60b6-aba9-48b2-a0c4-1a1de97c5982 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41481dc5a08a namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 20514e64-e3ba-4c0c-842c-ffd60215fda8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41481e3ad70b namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 62b5303c-1cab-4331-8710-2a04005d840d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41481e48e5b0 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 18084525-9cfa-45a3-a8f4-ba103149574c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.542s (2.542s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4148b5d00ad4 namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: aeb27307-2241-4caa-b968-c743c259de62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4148ba42f427 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: c91b59df-d207-45a4-9020-a4088c667cba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4148bab45131 namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 5eb63d97-6de5-419d-b643-1f36499e2a65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:32:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:00Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be414917bb34c4 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: a00faf64-3edc-4b01-a514-6f6768bde70f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4159306a829a namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 51f83e08-c676-4a31-bb8f-13376b67229c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:33Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be4159306af2fe namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 5fda5257-1160-495a-b758-e66a1b80b482 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: f9f04157-2a0a-4c48-93e8-7c17c1ee0b8a kind: Event lastTimestamp: "2026-07-01T19:32:35Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:35Z" name: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk.18be41599fe808b7 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: deb5cc55-b096-4da8-985b-6410484e3742 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-79b53-predictor-58774dd6b7 namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 1f0b92f9-5b99-462e-84d6-9b04ea0b6e4e kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: 'Created pod: error-404-isvc-79b53-predictor-58774dd6b7-tzcmk' metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:04Z" name: error-404-isvc-79b53-predictor-58774dd6b7.18be41444add0730 namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: ac8dd952-a527-440b-8be5-550dac5dfb8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: cbf50c55-de89-460f-bde4-d27aebd3b898 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Scaled up replica set error-404-isvc-79b53-predictor-58774dd6b7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:04Z" name: error-404-isvc-79b53-predictor.18be414448fadb60 namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 968eb888-077b-40bd-9b81-2317c2b101e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 7c5c7298-8903-4040-a411-97bd3d078521 kind: Event lastTimestamp: "2026-07-01T19:31:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:49Z" name: error-404-isvc-79b53-predictor.18be4147c761f096 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: dbc8195b-d105-42c0-bb47-c3c799b34159 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 7c5c7298-8903-4040-a411-97bd3d078521 kind: Event lastTimestamp: "2026-07-01T19:31:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:49Z" name: error-404-isvc-79b53-predictor.18be4147c7dd9ac7 namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: a56dbc46-81a1-4001-8bcc-e0986a9c08e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 7c5c7298-8903-4040-a411-97bd3d078521 kind: Event lastTimestamp: "2026-07-01T19:32:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:19Z" name: error-404-isvc-79b53-predictor.18be4152440fec05 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: baeb367b-da0a-4581-a913-a5bcea820dd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 7c5c7298-8903-4040-a411-97bd3d078521 kind: Event lastTimestamp: "2026-07-01T19:32:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:19Z" name: error-404-isvc-79b53-predictor.18be41524479c7bf namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: 780afffe-e62c-45c2-9ba4-318fe9dc0e54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-79b53 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ac2ccf7e-1bd1-41f5-a80d-2b000846d8b1 kind: Event lastTimestamp: "2026-07-01T19:32:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:10Z" name: error-404-isvc-79b53.18be414448ff8d5d namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 87abee44-480b-415f-9d9d-47de7f49dc15 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-79b53 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: ac2ccf7e-1bd1-41f5-a80d-2b000846d8b1 kind: Event lastTimestamp: "2026-07-01T19:32:10Z" message: InferenceService [error-404-isvc-79b53] is Ready metadata: creationTimestamp: "2026-07-01T19:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:10Z" name: error-404-isvc-79b53.18be4153d2d9ec33 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 48ffbd32-79e9-4b74-b42f-59ead0212734 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:24.134973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25257" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8003b-predictor-69459f667c-ml85h to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e55956340 namespace: kserve-ci-e2e-test resourceVersion: "25262" uid: f69dbd1a-fefe-4989-be7d-37af8914bad6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Add eth0 [10.132.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e6f5a97a3 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: 0c9e8314-c3b9-45d1-9291-70c87e33128a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e70a31cea namespace: kserve-ci-e2e-test resourceVersion: "25297" uid: 44842bc6-7eae-46f1-973b-a9807a735078 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e735621fb namespace: kserve-ci-e2e-test resourceVersion: "25299" uid: 1da7e482-834b-4367-88d4-cdf83c82b63e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e73c7e50c namespace: kserve-ci-e2e-test resourceVersion: "25300" uid: 814b19ca-639c-4963-998f-966ad3e24094 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e73d27db0 namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: a06b1a64-f0cc-4021-aa54-09467e3cb7d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e76f3e4de namespace: kserve-ci-e2e-test resourceVersion: "25302" uid: 8738c8c4-6977-4da1-8549-2d1342ca1604 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e77621f56 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: a21bc638-3073-487b-a92a-d70e9e327ea0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:53:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:00Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be426e7ba7a00d namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: a26accbe-94dc-4dd3-b73e-79b3529d26ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be427eb4a0c8b3 namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: 6ee06c83-3f7c-4a38-8f76-ab1681a62aa8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be427eb4a16d12 namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 4a19f017-aacf-4f62-ba38-369c7379f402 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8003b-predictor-69459f667c-ml85h namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 1255d853-0fd5-4a41-9546-ddd0cabb2554 kind: Event lastTimestamp: "2026-07-01T19:53:35Z" message: 'Readiness probe failed: Get "https://10.132.0.51:8643/healthz": dial tcp 10.132.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:35Z" name: error-404-isvc-8003b-predictor-69459f667c-ml85h.18be427f03f20cb0 namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 862e5fd9-0a69-4c34-a970-64b411f794bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8003b-predictor-69459f667c namespace: kserve-ci-e2e-test resourceVersion: "25248" uid: 4f09143f-29af-49c9-8f23-27bc3fb9c9b1 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Created pod: error-404-isvc-8003b-predictor-69459f667c-ml85h' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor-69459f667c.18be426e54e9d94c namespace: kserve-ci-e2e-test resourceVersion: "25259" uid: fdee9e81-1496-463a-ae63-6033b237db80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: 3a2a0693-523d-4b7a-8549-3f1795b1ec1d kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Scaled up replica set error-404-isvc-8003b-predictor-69459f667c from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b-predictor.18be426e52889f15 namespace: kserve-ci-e2e-test resourceVersion: "25249" uid: d7449c7c-9f0c-4dd0-bdc5-ad234bbae254 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 60aaa72c-9748-4ccf-be82-e02b8f91ca85 kind: Event lastTimestamp: "2026-07-01T19:52:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:39Z" name: error-404-isvc-8003b-predictor.18be4271d0e217fb namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 2d9ebcf8-88db-4970-a5de-6cf2dc003968 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 60aaa72c-9748-4ccf-be82-e02b8f91ca85 kind: Event lastTimestamp: "2026-07-01T19:52:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:39Z" name: error-404-isvc-8003b-predictor.18be4271d1433aae namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: 3b387a2c-a8cb-42bd-a513-b4d080c88084 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 60aaa72c-9748-4ccf-be82-e02b8f91ca85 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: error-404-isvc-8003b-predictor.18be42754ffe483b namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: b06e50ca-dee3-4ae4-9e58-550e7443117f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 60aaa72c-9748-4ccf-be82-e02b8f91ca85 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: error-404-isvc-8003b-predictor.18be4275508c8571 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 5aaee401-d38b-4b87-9ee8-037d5afe11c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: 71515a3f-1f83-41f4-a296-e8b5e8528d0c kind: Event lastTimestamp: "2026-07-01T19:53:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:10Z" name: error-404-isvc-8003b.18be426e528806b4 namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: b7753264-41b1-44a9-9ecf-b30f2a8f53f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: 71515a3f-1f83-41f4-a296-e8b5e8528d0c kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-8003b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8003b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b.18be426e52f7004f namespace: kserve-ci-e2e-test resourceVersion: "25253" uid: cd59a6ea-d0c3-4b66-a4ee-0d7ef8eeb76c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: 71515a3f-1f83-41f4-a296-e8b5e8528d0c kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8003b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:24Z" name: error-404-isvc-8003b.18be426e52f7657c namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: d9d61e5b-a235-4c8f-b03a-f3b9175c7d27 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 71515a3f-1f83-41f4-a296-e8b5e8528d0c kind: Event lastTimestamp: "2026-07-01T19:53:10Z" message: InferenceService [error-404-isvc-8003b] is Ready metadata: creationTimestamp: "2026-07-01T19:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:10Z" name: error-404-isvc-8003b.18be4279368e7d97 namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: ee0f6772-6f31-4acc-908f-323d276e8410 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:14.085813Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d109c-predictor-5d9b9bf885-t4frr to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162914fd3a2 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: d8098822-11c1-4ff8-a714-fb4daa08e04f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162abc5934f namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 929ec532-d1d2-46ef-8049-e0776c61c7dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162ad2c6edc namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 4db910c2-407f-4dfe-9376-b9bb15396456 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162afa5446e namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: e36ccd59-601f-4709-8953-23a9d4b480e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162b01631e4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 985d1f3e-1614-4a98-8122-153711e43631 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162b01f18c4 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 90613fe4-c646-456f-b4f8-ac41f00e84b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162b2e94f1b namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 9202ace9-a27a-4044-8924-e0adb530a570 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be4162b3592181 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 083576ff-3a0e-4d1f-8ad1-f592a22b647b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:33:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:52Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be41630fec45cc namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 3a945a7f-b962-47b4-aa4c-90309433c799 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be41e3b582a8a5 namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: 47c641cc-c59b-4e60-ba3c-6e06c5a27085 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8a910e61-15e1-46c0-82da-d2781902c582 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr.18be41e3b583f6f5 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: cd014a01-6164-4734-a4a6-4b672fa7a4d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d109c-predictor-5d9b9bf885 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: e929519f-4378-4d53-a215-103b6ff53d7a kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: 'Created pod: error-404-isvc-d109c-predictor-5d9b9bf885-t4frr' metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor-5d9b9bf885.18be4162903d8b46 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: bb12767f-d1f6-42bf-b934-ae2a8da7de22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 2e52b124-91b1-4c63-a425-b5177032c73a kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Scaled up replica set error-404-isvc-d109c-predictor-5d9b9bf885 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:14Z" name: error-404-isvc-d109c-predictor.18be41628d07b2cd namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 810beca6-c369-4ca1-8ea2-e213c2d5d2d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 02d3b31c-fdd2-4a40-b915-37420a152bf7 kind: Event lastTimestamp: "2026-07-01T19:33:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:59Z" name: error-404-isvc-d109c-predictor.18be41660bea5189 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 806bcee7-1628-4dba-ad13-df0ab66c8f2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 02d3b31c-fdd2-4a40-b915-37420a152bf7 kind: Event lastTimestamp: "2026-07-01T19:33:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:59Z" name: error-404-isvc-d109c-predictor.18be41660c6cc2e7 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 2c819db4-ddd8-4096-8bcc-04805d5df9c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 02d3b31c-fdd2-4a40-b915-37420a152bf7 kind: Event lastTimestamp: "2026-07-01T19:34:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:29Z" name: error-404-isvc-d109c-predictor.18be4170888fb350 namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: f5836183-b05b-4914-bcf4-e547a85d40a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 02d3b31c-fdd2-4a40-b915-37420a152bf7 kind: Event lastTimestamp: "2026-07-01T19:34:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:29Z" name: error-404-isvc-d109c-predictor.18be417088f693b3 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: ef41e00e-5d5b-414a-a7fb-921695047165 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d109c namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: a80fa7cb-ca96-4861-8528-9f8e1e62d5a9 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:02Z" name: error-404-isvc-d109c.18be41628d112428 namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: 9a03aad0-3e84-43d2-aff5-5d89ae560cc3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d109c namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: a80fa7cb-ca96-4861-8528-9f8e1e62d5a9 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: InferenceService [error-404-isvc-d109c] is Ready metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:02Z" name: error-404-isvc-d109c.18be416dcc367ca6 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: de3e3d1d-8089-4a4d-bbb5-2ac70c5a6af6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:39.224309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f705b-predictor-d6ff57dff-d2dhp to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f41e789f55 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: b92694ad-d3b7-4b7b-b920-2cd1994dfd98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f43860b9af namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: b394b195-451c-4076-9d0e-682e2bd63504 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f43a50fb01 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: f8cfd83c-d1ee-4074-992c-03cf6f87359b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f43c9f43d9 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: caec4408-c5ec-44a5-94f0-c311f6a0c676 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f43d0ecf0b namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 36591dea-fa72-48c9-ab21-c7c1261ad304 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f43d185f48 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: daf03eb9-9229-4618-a9c4-cb4cda20fdac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f44026d898 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 95f2d84f-8cb2-4a53-8fdb-4e07f5eabcc4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f44096e154 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 9e61adde-c787-483c-824e-572f52209c32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:52:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:56Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be41f45d10b47e namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 2827a865-045d-4c7b-98f1-f23d3efa4a9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:53Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be4275479538df namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: 93f5e3ea-a739-42be-9653-155579c3ae78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:53Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be427547957d16 namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 01bbca8b-3426-444a-9387-f83180cfa7f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f7e25b7d-d3e1-47cd-a289-1e3df644af1c kind: Event lastTimestamp: "2026-07-01T19:52:56Z" message: 'Readiness probe failed: Get "https://10.132.0.49:8643/healthz": dial tcp 10.132.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:56Z" name: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp.18be4275d1940de8 namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: b50acb79-5f23-4ead-8eb2-35ec5b21fb07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f705b-predictor-d6ff57dff namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 0c4cfd5a-503e-4caa-910b-e600fe3e35ef kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created pod: error-404-isvc-f705b-predictor-d6ff57dff-d2dhp' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor-d6ff57dff.18be41f41dfedee7 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 262332e2-5c6d-466e-9987-7ae5af12af2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 6f208e51-75e5-4c97-85d9-89580c974a0f kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Scaled up replica set error-404-isvc-f705b-predictor-d6ff57dff from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b-predictor.18be41f41bd2f746 namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: ef3eb328-0578-42f8-b8eb-6a7621746cfb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 7308b7c4-3714-4df3-ae7f-b7214b6bceb4 kind: Event lastTimestamp: "2026-07-01T19:43:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:54Z" name: error-404-isvc-f705b-predictor.18be41f79a57e861 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: f36db19b-391a-456d-875f-2da7deb372cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 7308b7c4-3714-4df3-ae7f-b7214b6bceb4 kind: Event lastTimestamp: "2026-07-01T19:43:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:54Z" name: error-404-isvc-f705b-predictor.18be41f79ad19924 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: ea51d925-393b-4f5a-9937-d46bb75262aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 7308b7c4-3714-4df3-ae7f-b7214b6bceb4 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: error-404-isvc-f705b-predictor.18be41fb1a1b7980 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: bba7be3c-d62b-4627-833e-52a51a20c7ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 7308b7c4-3714-4df3-ae7f-b7214b6bceb4 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: error-404-isvc-f705b-predictor.18be41fb1a83ba20 namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: 50e4c8e2-71bf-4e96-9da7-0e50bf3d7fd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e7a3411b-c6b9-40f1-a790-c38b82968f1e kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-f705b.18be41f41bd1a98c namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: c863ebb0-43fa-4968-828d-c3eeb0cabf05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e7a3411b-c6b9-40f1-a790-c38b82968f1e kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-f705b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f705b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b.18be41f41c94855d namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 660b68a6-3fcc-4d89-badb-214ec8527ee3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e7a3411b-c6b9-40f1-a790-c38b82968f1e kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f705b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f705b.18be41f41c94cd47 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: b3648375-ca49-4a4d-8792-540ce8e53658 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: e7a3411b-c6b9-40f1-a790-c38b82968f1e kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: InferenceService [error-404-isvc-f705b] is Ready metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-f705b.18be41ff189a1db4 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: f028d450-159b-4dac-8e25-07b8f3a7cd54 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:28.908694Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7f91-predictor-69c44d6649-89l57 to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e3bf5519dd namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 7c7470f5-fdca-4389-9873-505fa3fe8a81 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f7f91-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e3cbaf506b namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: a12c3925-db94-41b2-a898-25771861a0e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:29Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e3fd6361c4 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 2542a823-b92b-410b-9324-8943e41f7024 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e3fec97e1e namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 216f3003-b93d-4432-a942-35a42e624d17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:30Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e401d9b573 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: ebe0838c-aef5-418e-94c4-f0db391fd9b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:30Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e4024b5a73 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: d96dcf41-982b-490e-bd93-a1117f9b2d80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:30Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e40255ff64 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 202f4fa3-6072-464d-8f5d-70d3d15e4436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:30Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e4055ba728 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: b9a4cebc-8fd7-4c8f-beac-ff9ea3fcd250 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:42:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:30Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e405d01cc5 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 65630ccd-17b2-479e-a26a-ea4391387514 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:43:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:07Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41e436396ec1 namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: fb535e17-1b38-43e6-af74-88aa9f609570 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41f413c307da namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 5123eb64-ee34-402b-82e8-3ba81b7ac462 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7f91-predictor-69c44d6649-89l57 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 492f1bb4-3f05-44d8-bb47-1a1b044cf510 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: error-404-isvc-f7f91-predictor-69c44d6649-89l57.18be41f413c6dc3d namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: 805b8ef8-d282-45d7-8d58-93c8902aa616 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7f91-predictor-69c44d6649 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: d783fed3-0cf1-4ffe-9a9b-e80fd4f786c8 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'Created pod: error-404-isvc-f7f91-predictor-69c44d6649-89l57' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-f7f91-predictor-69c44d6649.18be41e3bedd58c6 namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: 022e7e09-14cd-40fe-ba0e-64adf43a7872 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: ea2c9dff-6bec-4cb3-9248-e2c454de66ff kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Scaled up replica set error-404-isvc-f7f91-predictor-69c44d6649 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-f7f91-predictor.18be41e3bd53c57e namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 0b884ed3-e09e-4be0-ab01-bda683cd578c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 313de1ef-5b08-4654-a09b-d8a89baaeece kind: Event lastTimestamp: "2026-07-01T19:43:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:13Z" name: error-404-isvc-f7f91-predictor.18be41e73bc1eb47 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 5c289777-9fa5-4afb-927d-303ed741d349 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 313de1ef-5b08-4654-a09b-d8a89baaeece kind: Event lastTimestamp: "2026-07-01T19:43:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:13Z" name: error-404-isvc-f7f91-predictor.18be41e73c2f98f6 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: e51c4d76-6100-49bf-bf70-0b665c906072 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 313de1ef-5b08-4654-a09b-d8a89baaeece kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: error-404-isvc-f7f91-predictor.18be41f1b8da2b80 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: dd1bb0b0-5244-4823-952f-df52d65825c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 313de1ef-5b08-4654-a09b-d8a89baaeece kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: error-404-isvc-f7f91-predictor.18be41f1b95d179c namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: c9c64cb9-0b8d-4987-848e-be8a3ed4f7b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: b60983b8-4814-4798-af43-49f15b94e757 kind: Event lastTimestamp: "2026-07-01T19:43:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:18Z" name: error-404-isvc-f7f91.18be41e3bd57e9fd namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 98abecac-acaf-49ce-9ae8-9406254ad0f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: b60983b8-4814-4798-af43-49f15b94e757 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7f91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-f7f91.18be41e3bdcb1305 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 8217907e-cc82-43dc-98b1-62d752c79347 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: b60983b8-4814-4798-af43-49f15b94e757 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:28Z" name: error-404-isvc-f7f91.18be41e3bdcb5db9 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 0d8c8d81-76e6-4c58-a9ab-2ad2672e320b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: b60983b8-4814-4798-af43-49f15b94e757 kind: Event lastTimestamp: "2026-07-01T19:43:18Z" message: InferenceService [error-404-isvc-f7f91] is Ready metadata: creationTimestamp: "2026-07-01T19:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:18Z" name: error-404-isvc-f7f91.18be41ef2d783868 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 794f167c-f83a-4382-9933-2aa28ce6cf9e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:03.701438Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w to ip-10-0-139-28.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:03Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be414435cc22b1 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: a0495d02-71c0-4a80-8a91-5cc91fe0aa18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be414450d05c75 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 02c2c16b-17ba-4889-9783-452b0b41d8bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:530addec2bc67307a961138e3fb6a7ef47f6fd87e335859b392a260fb4052870" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4144527b298a namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: cb6204fc-8b7c-4d1e-9e5c-dca4d00220e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be414455a531f9 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 10c6b2e8-3f87-43d0-af1c-591ea70b721b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4144561da907 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 4c1c6246-d718-4acc-8ee6-2aa4db8c9c7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:07Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1693" metadata: creationTimestamp: "2026-07-01T19:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:07Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be41452c6ecd57 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 9b2f41de-7ab3-4836-8d3a-056d1638f327 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1693" in 15.608s (15.608s including waiting). Image size: 1555846067 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4148cec40c5f namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 7f800735-030e-4191-92d5-462121ef4df9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4148d2ae8bf2 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: b60ba8f5-90f9-4495-bdcc-450ee48a28c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4148d32fab9f namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 27884b21-3f23-498e-ab08-783ddb282ac0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4148d33eae44 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: befa5673-29c9-4515-8edb-dad45c06bfa6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.373s (2.373s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be414960b3203b namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: f0420f93-2149-482b-8b39-5f405faff2f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be414963d5d14b namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: a1290450-1d00-4e4a-8eec-4568c78c7fa7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4149646b4fb6 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 91065611-75e3-4cf1-b215-4c27cd788c8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:32:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:32Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be4149a0da44ff namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 8e23a4bf-0792-47c1-8e69-15a26975004b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be41628a165b28 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 0f4c3dcb-7edd-4442-bb06-f38eec24a20a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be41628a170c3f namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 8444a79b-02b1-44de-b60a-718848155374 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b355a727-3563-42f4-825e-b18cfe2c9369 kind: Event lastTimestamp: "2026-07-01T19:33:17Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:17Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w.18be41637930cbcf namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: b205c7af-e281-41d6-adaf-d41a2f11c567 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6ccb86d5d8 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 45adc420-3778-4118-8b98-161b39fdca61 kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6ccb86d5d8-8kf9w' metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: isvc-sklearn-graph-1-predictor-6ccb86d5d8.18be4144353837d0 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 7afd8262-02f3-44bf-b9c5-6b58b109e5a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: c5cd254c-dfb5-4876-bab7-3a8e92ff09c3 kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6ccb86d5d8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: isvc-sklearn-graph-1-predictor.18be41443060a571 namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: c0498f7b-0d9a-4349-9401-564b9b487f11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 212a3e02-362c-4735-bade-02e7807cb80e kind: Event lastTimestamp: "2026-07-01T19:32:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:03Z" name: isvc-sklearn-graph-1-predictor.18be4147b5ba1ebe namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 34047c87-3ee7-44c3-aa53-6282c41692e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 212a3e02-362c-4735-bade-02e7807cb80e kind: Event lastTimestamp: "2026-07-01T19:32:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:03Z" name: isvc-sklearn-graph-1-predictor.18be4147b6293f3c namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: c7c6ac6a-0ed2-4f79-b65b-713d0415dcef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 212a3e02-362c-4735-bade-02e7807cb80e kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: isvc-sklearn-graph-1-predictor.18be414eb416b056 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 343e39bb-ca30-4b87-8f0b-0b43a3a0366e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 212a3e02-362c-4735-bade-02e7807cb80e kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: isvc-sklearn-graph-1-predictor.18be414eb476f8de namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 338f4c26-15eb-4e05-887c-6916cc0826a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 9ece0ca6-e777-4811-af8f-652f1f1697b1 kind: Event lastTimestamp: "2026-07-01T19:32:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:43Z" name: isvc-sklearn-graph-1.18be4144366af75f namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 515224df-f6af-483a-9cfd-368f6bd6f4ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 9ece0ca6-e777-4811-af8f-652f1f1697b1 kind: Event lastTimestamp: "2026-07-01T19:32:43Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T19:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:43Z" name: isvc-sklearn-graph-1.18be415b58ff89fd namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 420125af-a1ba-49ae-b6de-ee8d979119bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:04.185995Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm to ip-10-0-139-28.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be414452adebb2 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 817a4005-8b21-4896-a556-31da6f8c32bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be41446c86e0ca namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: cd76ecec-c05f-4ce9-be1b-5c2ec60a22ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:530addec2bc67307a961138e3fb6a7ef47f6fd87e335859b392a260fb4052870" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be41446e238163 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: b05c7ce3-fd4b-477a-9cbb-45a2da84ecf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be414472485a4b namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: f535688f-5fea-4f22-9eca-574a06fe4e85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be414472de2513 namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: 32ee9646-586c-4f9e-8198-9f1559f61d22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1693" metadata: creationTimestamp: "2026-07-01T19:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:08Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4145689c373d namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 906fcfbf-5f40-47d6-a777-302a1763aca4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1693" in 14.598s (14.598s including waiting). Image size: 1555846067 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4148cec2c196 namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: c5d1f4e1-7b89-41e9-945d-d90819270ef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4148d2bd3b2a namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 5edca20b-a2e5-4c86-a3a8-a8a97137290b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4148d33ef1fc namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 3e0a5a09-77ed-48e2-9772-4dd5080fa7a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4148d34a643c namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 40a53bd5-58ec-4f1a-8c39-78ae2f6b21f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.362s (2.362s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4149601a4dda namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: f3b22d93-4572-4754-81b9-4d8862f6eccf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be414963b9321e namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 7d732203-d421-4208-9510-d8613009168a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:31:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:25Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be414964558312 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 01f7eaee-ed69-438f-b777-d085db02a95d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:32:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:32Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4149a0da86a0 namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 66b6e6f0-049a-4551-93f7-b2d4daad39c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4162862e4c98 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: cbfcd712-d545-4374-ad1e-cf393244ada4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be4162862ed2d8 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: b248c6cd-9f5e-49de-8a2d-73a3287acfad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9e8f5e0a-b6c7-468a-a6f0-cb6abf69fc17 kind: Event lastTimestamp: "2026-07-01T19:33:17Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:17Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm.18be41637931a6b5 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 11f89f44-06dd-4155-a01b-61988c464f51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-28.ec2.internal source: component: kubelet host: ip-10-0-139-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-76b45d9b44 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: d20ec9f8-83be-415b-b239-514e5e211ca5 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-76b45d9b44-l4hwm' metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor-76b45d9b44.18be4144520d0c67 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 2a9ce62c-05a4-4aa7-9cb1-6ea2079b6c0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: c46ad2f5-7982-44e9-9f73-99b25a965167 kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-76b45d9b44 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:04Z" name: isvc-sklearn-graph-2-predictor.18be41444ee1c16f namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: f0e9c2d6-48df-48c1-94b4-303c4e0a178e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 2d316ccc-0824-4fd3-bcca-d052f0511c30 kind: Event lastTimestamp: "2026-07-01T19:31:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:34Z" name: isvc-sklearn-graph-2-predictor.18be4147cd4ec897 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: c1e37444-6ca0-4632-bb22-446e32ddd0dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 2d316ccc-0824-4fd3-bcca-d052f0511c30 kind: Event lastTimestamp: "2026-07-01T19:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:34Z" name: isvc-sklearn-graph-2-predictor.18be4147cdb46da3 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 3cc01c8a-17ff-4199-8586-946ae800c888 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 2d316ccc-0824-4fd3-bcca-d052f0511c30 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:04Z" name: isvc-sklearn-graph-2-predictor.18be414ecb241301 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: f5de317b-a910-402f-810a-031e26c31268 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 2d316ccc-0824-4fd3-bcca-d052f0511c30 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:04Z" name: isvc-sklearn-graph-2-predictor.18be414ecbb1df98 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: e89fd252-d370-436d-aa36-e163e890583c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: a0c87b73-61c9-494e-a435-d9dc7be3a0c1 kind: Event lastTimestamp: "2026-07-01T19:32:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:43Z" name: isvc-sklearn-graph-2.18be41444ee27539 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 262da8a1-ff67-4bec-a70a-221b08f62321 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: a0c87b73-61c9-494e-a435-d9dc7be3a0c1 kind: Event lastTimestamp: "2026-07-01T19:32:43Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T19:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:43Z" name: isvc-sklearn-graph-2.18be415b5b928d11 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 4631ff57-a7fb-4550-9413-f97af242c2b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:03.971854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-zkcnx to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414445ea5f66 namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: bbf8d03a-4963-4d55-a90b-eb8093372b9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be41445fa9126f namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 2c350736-10b1-4f4f-94c1-ec8723295fde reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:530addec2bc67307a961138e3fb6a7ef47f6fd87e335859b392a260fb4052870" metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be4144613b1376 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 23aef428-93f6-4463-8584-0f60c4883a94 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:530addec2bc67307a961138e3fb6a7ef47f6fd87e335859b392a260fb4052870" in 3.444s (3.444s including waiting). Image size: 300562904 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be41452e8b13e6 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: f0dfe224-7432-4095-ad5d-2be4a0a1b5f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414532386d8c namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 2296be21-36f8-4e85-9212-33d4cbdb1a52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414532aea3ea namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: e3a28862-edfa-4815-820c-fe3a63acc9d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:12Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be4146282c338f namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 851a513f-97cc-4312-99ac-a9340f8a8ed6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.607s (20.607s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414af47b4c6b namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 616b14c7-f208-4b5f-896e-93ff7d2b936e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414af9c07660 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: b9fa4a48-5051-41d3-a50c-096ffabba312 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414afa7adc5f namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: de161ee7-c812-4332-a7fc-ba1c9295977e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414afa880803 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 1ea5f795-4827-41f9-b25b-928d4a66df43 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:34Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.157s (2.157s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414b7b21950e namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: 81f51a7f-82ff-495c-aaf8-08eed18c407f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414b7e6abaf7 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: c96eb4cc-4ade-48a4-8b11-836c48b81a94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:31:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414b7eddf29a namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 8dab1b5b-812a-4772-9fd3-94c278da4451 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T19:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be414b88a53023 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: ffd520e8-9d74-4eec-b81d-7dea09cb49c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be41628226e46f namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 733623e2-5d04-44bc-8f4a-31d761bfd3e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be416282275116 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 7e1fda88-3ad4-4f5a-b92b-3d3564b219a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 3e060e8e-2be5-4096-b4f0-7535a1e83d3e kind: Event lastTimestamp: "2026-07-01T19:33:16Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-zkcnx.18be41630ce3e8a7 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: fbc45a7a-19ff-4f42-adb4-eecbeb5edb12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 3e644652-cae0-4a1f-83d2-08c42dbbcf0d kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-zkcnx' metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be41444582f9ba namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 80843005-3e45-485a-a3f1-f0fa8c910768 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: a8be3e9f-ea2b-41b7-8e82-2ba277365f20 kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: isvc-xgboost-graph-predictor.18be4144427223c9 namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 2a0a6935-f567-423a-9f1f-fd23317b9f8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 326fbb59-b0a4-46ac-a8f1-d7c008868521 kind: Event lastTimestamp: "2026-07-01T19:32:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:03Z" name: isvc-xgboost-graph-predictor.18be4147c0f42992 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: d968a04e-5be8-474a-bc4a-f043d762ebb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 326fbb59-b0a4-46ac-a8f1-d7c008868521 kind: Event lastTimestamp: "2026-07-01T19:32:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:03Z" name: isvc-xgboost-graph-predictor.18be4147c160df24 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 86494241-e66e-43eb-889d-81692a496eee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14678" uid: 326fbb59-b0a4-46ac-a8f1-d7c008868521 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:04Z" name: isvc-xgboost-graph-predictor.18be4155bc14d551 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 03d91330-01ab-4c14-adad-d3f2d8f247e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14678" uid: 326fbb59-b0a4-46ac-a8f1-d7c008868521 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:04Z" name: isvc-xgboost-graph-predictor.18be4155bc893ce1 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: e3487469-745b-4a59-872f-ba66d843a4af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 9f54729b-2de2-4a97-8307-085ac7e66807 kind: Event lastTimestamp: "2026-07-01T19:32:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:41Z" name: isvc-xgboost-graph.18be4144427666ad namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 09570add-d01b-4f82-a677-13591b08531d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 9f54729b-2de2-4a97-8307-085ac7e66807 kind: Event lastTimestamp: "2026-07-01T19:32:41Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T19:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:41Z" name: isvc-xgboost-graph.18be415aec2ddae9 namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: c3a9778f-e2a6-483a-93c2-0b0212632558 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:03.622643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7988dc58f6-ltwnw to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:03Z" name: model-chainer-7988dc58f6-ltwnw.18be416021a87563 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: 3c8c5fe0-0a4e-4166-9e8d-cd3e1f0ff6c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:03Z" name: model-chainer-7988dc58f6-ltwnw.18be41602fad8440 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: e677199e-8c64-4032-997a-615f18654e56 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:04Z" name: model-chainer-7988dc58f6-ltwnw.18be41605ed9189a namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: f1f650d7-66f1-4e52-8952-16d31d77985f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:04Z" name: model-chainer-7988dc58f6-ltwnw.18be416060043267 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 908fc6e6-6efe-4955-b3f7-120d4f01f90a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:04Z" name: model-chainer-7988dc58f6-ltwnw.18be416062866002 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 3adb3d36-8bc0-49d9-9dd6-c415bec3d2d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:04Z" name: model-chainer-7988dc58f6-ltwnw.18be416062f9ef20 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: ce94aa6f-73ed-461e-abcf-489a9b6f7fad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:13Z" name: model-chainer-7988dc58f6-ltwnw.18be41627c9c78bd namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 0fd67621-0094-4555-9d36-f811cb11450a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7988dc58f6-ltwnw namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: e268b723-faea-4a65-a7ba-857db63861a3 kind: Event lastTimestamp: "2026-07-01T19:33:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:41Z" name: model-chainer-7988dc58f6-ltwnw.18be41631ffe6de4 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: c4f87c5f-01de-49cd-abd3-a3de31531a36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7988dc58f6 namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 355286e6-f5e1-4cf3-be4b-871c003c3632 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'Created pod: model-chainer-7988dc58f6-ltwnw' metadata: creationTimestamp: "2026-07-01T19:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: model-chainer-7988dc58f6.18be416021293f91 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: f6cabe27-d053-4498-b5b6-f87855799721 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 645c246e-b7b1-419c-a588-7538b5cbdf59 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: Scaled up replica set model-chainer-7988dc58f6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: model-chainer.18be41602052fb56 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 0899cc12-b3a2-4ebd-844a-0ddfb6eae2db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 38aa12c5-f734-4ed9-b1fe-f12e216e4fa3 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T19:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:03Z" name: model-chainer.18be416025714fc5 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: e23aaa1b-64b9-471b-989d-2856661136b2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:34:13.906025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d109c-6ddb5c8b58-56ppr to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:34:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:34:13Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be41707edfc9a6 namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: bb5e4881-3a9e-419c-82af-dbf4a263da31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:14Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:34:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-d109c-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:14Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be41708abce3f9 namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 164d6014-6374-473a-8c1f-72e404669e09 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:14Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:34:14Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:34:14Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be4170bbdd954a namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: fd630a55-5f02-4ff9-8340-2d6ab1632ad4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d109c} kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:34:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:14Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be4170bd1c484b namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 34200881-2d51-4a5d-b18d-9886013a2b70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d109c} kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:34:15Z" message: 'Created container: sequence-graph-d109c' metadata: creationTimestamp: "2026-07-01T19:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:15Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be4170c02952d0 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 40c55cae-b4b4-4626-8920-7101abd5fdc7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d109c} kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:34:15Z" message: Started container sequence-graph-d109c metadata: creationTimestamp: "2026-07-01T19:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:15Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be4170c096cd95 namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 41b33e98-ad9d-4e8b-8a77-ed5e2686365e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d109c} kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Stopping container sequence-graph-d109c metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:28Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be41e3b019b496 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 6199f792-78f7-4110-92f3-7850a43ef118 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d109c} kind: Pod name: sequence-graph-d109c-6ddb5c8b58-56ppr namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 0eed7b11-4b52-4838-a9b3-103c34b46928 kind: Event lastTimestamp: "2026-07-01T19:42:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:56Z" name: sequence-graph-d109c-6ddb5c8b58-56ppr.18be41e4678cd59d namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 3137c4a0-dcce-4e3c-9a27-9f908a53f333 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d109c-6ddb5c8b58 namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 33efe3bc-bb23-4a0f-85b8-d75efb2b2e76 kind: Event lastTimestamp: "2026-07-01T19:34:13Z" message: 'Created pod: sequence-graph-d109c-6ddb5c8b58-56ppr' metadata: creationTimestamp: "2026-07-01T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:13Z" name: sequence-graph-d109c-6ddb5c8b58.18be41707e68d2a3 namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: 92836115-f011-476f-96e4-f3194d8d51fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 28cbf178-f8de-418b-b863-af7593b0ca26 kind: Event lastTimestamp: "2026-07-01T19:34:13Z" message: Scaled up replica set sequence-graph-d109c-6ddb5c8b58 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:13Z" name: sequence-graph-d109c.18be41707d70d7bb namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: b5996d44-b3be-4701-81c4-b523869111a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: f1a02008-f9d4-4b00-8954-bf7652472019 kind: Event lastTimestamp: "2026-07-01T19:34:13Z" message: InferenceGraph [sequence-graph-d109c] is Ready metadata: creationTimestamp: "2026-07-01T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:13Z" name: sequence-graph-d109c.18be417083567f98 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 8dcecf10-7e34-4a17-9c46-f2ef33f08e0e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 50844eaf-1a79-46cb-b477-397bf27d906a kind: Event lastTimestamp: "2026-07-01T19:34:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:28Z" name: sequence-graph-d109c.18be4173fbf4776e namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 845cec8b-3111-432d-a888-2a6d6540602d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 50844eaf-1a79-46cb-b477-397bf27d906a kind: Event lastTimestamp: "2026-07-01T19:34:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:28Z" name: sequence-graph-d109c.18be4173fc5bd1a9 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: ac269186-e283-4b31-bcfe-69160380da0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 50844eaf-1a79-46cb-b477-397bf27d906a kind: Event lastTimestamp: "2026-07-01T19:34:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:43Z" name: sequence-graph-d109c.18be41777b1e8995 namespace: kserve-ci-e2e-test resourceVersion: "16952" uid: 2a710581-4119-4dcd-b80b-28c663136b00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d109c namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 50844eaf-1a79-46cb-b477-397bf27d906a kind: Event lastTimestamp: "2026-07-01T19:34:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:43Z" name: sequence-graph-d109c.18be41777b842e60 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: aa3ee01d-4ce7-4001-81dc-5d43062a4303 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:39.166188Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f705b-5fd9c8dfb8-sxjr4 to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:39Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be42021348c1a3 namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 3522e82e-77e9-4dde-bfa8-a24b74bf3bd5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f705b-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:39Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be42021fb2a8e5 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 2a572dc1-73c4-4854-880a-23ac7189623f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:44:40Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:40Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be42025040aefd namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 04afc802-461b-4b58-8528-438b7dda9c37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f705b} kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:44:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:40Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be4202518d9f22 namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 9e02db83-e1fd-4ade-9206-1c471d64be2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f705b} kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:44:40Z" message: 'Created container: sequence-graph-f705b' metadata: creationTimestamp: "2026-07-01T19:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:40Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be420254c8ecd6 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 6749e0d2-0c09-4ae0-98bd-0cd90d8a05a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f705b} kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:44:40Z" message: Started container sequence-graph-f705b metadata: creationTimestamp: "2026-07-01T19:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:40Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be42025539e60f namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: e716aabb-59b3-4f21-b4fb-ca0e8b80b9e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f705b} kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: Stopping container sequence-graph-f705b metadata: creationTimestamp: "2026-07-01T19:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:53Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be427542617a74 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: fd27162a-59c3-46dc-9773-6f6a566a8998 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f705b} kind: Pod name: sequence-graph-f705b-5fd9c8dfb8-sxjr4 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bcf6e0c0-cb7b-45b3-8dc6-513a58ccf8ac kind: Event lastTimestamp: "2026-07-01T19:53:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:22Z" name: sequence-graph-f705b-5fd9c8dfb8-sxjr4.18be427631ecc991 namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 737a0396-4317-4719-a1a6-7fb744c4793e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f705b-5fd9c8dfb8 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: d50a3747-8189-424d-9cb2-54116f5327e6 kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'Created pod: sequence-graph-f705b-5fd9c8dfb8-sxjr4' metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: sequence-graph-f705b-5fd9c8dfb8.18be420212d5b99b namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 2ee9b4f0-236d-42bf-93d6-df97692ddaf0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: ea2a8a78-dcd0-4db2-8875-4fe42aa97137 kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: Scaled up replica set sequence-graph-f705b-5fd9c8dfb8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: sequence-graph-f705b.18be42021226a241 namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: e642789e-9283-4886-8983-5e3dc55b721e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 46532153-5d45-4bb4-a8e9-46b2c59ee63c kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: InferenceGraph [sequence-graph-f705b] is Ready metadata: creationTimestamp: "2026-07-01T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:39Z" name: sequence-graph-f705b.18be4202180d1ed2 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: 25a0b0cd-14d2-41c2-91fb-1755f3ad12c3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 55b14c89-68e5-469d-bc53-d2b661967f11 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: sequence-graph-f705b.18be42059073d265 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: e659f63d-3757-4d56-9828-9888f0984ebf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 55b14c89-68e5-469d-bc53-d2b661967f11 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: sequence-graph-f705b.18be420590dd4410 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: 585ad145-61bf-4e06-ade7-0f7389e40562 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 55b14c89-68e5-469d-bc53-d2b661967f11 kind: Event lastTimestamp: "2026-07-01T19:45:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:09Z" name: sequence-graph-f705b.18be42090f9ee678 namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 05fae1b1-f5eb-4354-9a20-449d3dfa1f1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f705b namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 55b14c89-68e5-469d-bc53-d2b661967f11 kind: Event lastTimestamp: "2026-07-01T19:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:09Z" name: sequence-graph-f705b.18be42091008f0a5 namespace: kserve-ci-e2e-test resourceVersion: "22218" uid: 2fde745f-24ca-4af7-bee8-124f5280cc31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:28.829010Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f7f91-5c7b545686-vfvqz to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:28Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1b2dc88cb namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 2db34951-8ae5-4baa-b174-bdd0d1d307b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f7f91-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:28Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1b93c8261 namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: ae2746f1-d44f-4779-8097-87b2cce04e5d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:29Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1ddbff147 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 11df4c3d-fa4b-429e-ae12-9077128891b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7f91} kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:29Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1df1ef469 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 730c67db-f90d-453e-88df-9807933f399a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7f91} kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: 'Created container: sequence-graph-f7f91' metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:29Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1e20679b3 namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 566b4a79-3cdf-4ff6-bd36-999e196b190b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7f91} kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:29Z" message: Started container sequence-graph-f7f91 metadata: creationTimestamp: "2026-07-01T19:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:29Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f1e287b54b namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: bc61575e-62cf-4401-b4c1-b057b838801a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7f91} kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:43:38Z" message: Stopping container sequence-graph-f7f91 metadata: creationTimestamp: "2026-07-01T19:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:38Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f40d4e1d5c namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: f35fe2a1-3a46-4418-b9ee-2bd41d46460b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7f91} kind: Pod name: sequence-graph-f7f91-5c7b545686-vfvqz namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 55a335f9-b53e-4de6-a17e-28b5ebb0e737 kind: Event lastTimestamp: "2026-07-01T19:44:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:06Z" name: sequence-graph-f7f91-5c7b545686-vfvqz.18be41f4ae77e423 namespace: kserve-ci-e2e-test resourceVersion: "21653" uid: 45a6812f-b4ec-43f7-988a-f7d05c8bb8be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f7f91-5c7b545686 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 83bca93e-9b60-4dce-854b-3ada5b1b5736 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'Created pod: sequence-graph-f7f91-5c7b545686-vfvqz' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: sequence-graph-f7f91-5c7b545686.18be41f1b256ccaa namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: c7774c0a-55a2-46be-92b4-9ae0764ac55b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: c99af1b9-281b-4036-b22b-3a5ec55a3af9 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: Scaled up replica set sequence-graph-f7f91-5c7b545686 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: sequence-graph-f7f91.18be41f1b1961277 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: e0d5d406-2b2f-4311-95c9-7548a4da0861 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: 0c118755-563e-4a9b-bae7-540c96d37da9 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: InferenceGraph [sequence-graph-f7f91] is Ready metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:28Z" name: sequence-graph-f7f91.18be41f1b73fa1ec namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 3631469d-6ea4-42d2-b97a-e6b00bc7d41f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:54:34.397807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5e51f-6b6dcfc757-955nw to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be428ca9da74d8 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 4e65be94-86aa-4672-a55f-586a9faa1741 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be428cc304ac5d namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: fe4faff4-059a-4d8c-9c71-4fd224a7f1f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5e51f} kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be428cc43121c7 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: db74d5f8-6b99-4156-b00b-da198304d940 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5e51f} kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: 'Created container: splitter-graph-5e51f' metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be428cc6f58a82 namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: b3bb7c05-a92a-4421-b534-c67e49306d38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5e51f} kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: Started container splitter-graph-5e51f metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be428cc762b295 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: d00ad23a-f016-4772-b8f3-cef42d72a4ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5e51f} kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T20:02:49Z" message: Stopping container splitter-graph-5e51f metadata: creationTimestamp: "2026-07-01T20:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:49Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be42ffd7962e67 namespace: kserve-ci-e2e-test resourceVersion: "30528" uid: d39b7306-3d4e-46de-b0bd-aa66c4bfb6d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5e51f} kind: Pod name: splitter-graph-5e51f-6b6dcfc757-955nw namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 6365385e-52e2-4056-8968-2ed2ba96f8ad kind: Event lastTimestamp: "2026-07-01T20:03:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:15Z" name: splitter-graph-5e51f-6b6dcfc757-955nw.18be430044589409 namespace: kserve-ci-e2e-test resourceVersion: "30762" uid: 0a89656e-5091-49e4-b0f9-5f0312289d40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5e51f-6b6dcfc757 namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 8a16c909-a355-480c-9981-5b8ffd1b8952 kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: 'Created pod: splitter-graph-5e51f-6b6dcfc757-955nw' metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f-6b6dcfc757.18be428ca955fb3a namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: dffeb9eb-cedd-45f4-8aee-0a80fb29e10a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 4a1b4884-92a7-4201-9ec3-a1f0a18cf6d0 kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: Scaled up replica set splitter-graph-5e51f-6b6dcfc757 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f.18be428ca8187a5d namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 49797bf7-b22f-4b50-b6b8-183d96f5d1a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: becde090-9d69-4187-ae6f-4abcddde7550 kind: Event lastTimestamp: "2026-07-01T19:54:34Z" message: InferenceGraph [splitter-graph-5e51f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:34Z" name: splitter-graph-5e51f.18be428cae2ea4fa namespace: kserve-ci-e2e-test resourceVersion: "27099" uid: cff7f625-b0a9-4bf1-8d2a-ef1d21d03401 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: f96574ff-6a8a-4457-9d91-f23c0aee9133 kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: splitter-graph-5e51f.18be42902686f52d namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: d35f0785-1268-41a7-8afc-9b6164ef4ba2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: f96574ff-6a8a-4457-9d91-f23c0aee9133 kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: splitter-graph-5e51f.18be429026f75cc4 namespace: kserve-ci-e2e-test resourceVersion: "27223" uid: 16e99562-86bf-4c44-985d-96f3f79f7d81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:55:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: f96574ff-6a8a-4457-9d91-f23c0aee9133 kind: Event lastTimestamp: "2026-07-01T19:55:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:04Z" name: splitter-graph-5e51f.18be4293a5a45120 namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: 58fb0ae5-262a-428c-8c26-21a7d84f27df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:55:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5e51f namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: f96574ff-6a8a-4457-9d91-f23c0aee9133 kind: Event lastTimestamp: "2026-07-01T19:55:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:04Z" name: splitter-graph-5e51f.18be4293a6096775 namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: aa6f86e2-5510-45d6-93a0-3d6defeb6c15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:24.052629Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-8003b-7dc6f58dd9-9kkzl to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427c48f433d6 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: e23f9708-7bca-4b38-8088-d5d14dee4830 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427c62299541 namespace: kserve-ci-e2e-test resourceVersion: "26289" uid: 33af0f5d-6ffe-4a80-8a57-362649c4ac1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8003b} kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427c63608be9 namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: daf40421-9d7f-459f-83d3-c645de413f46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8003b} kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'Created container: splitter-graph-8003b' metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427c660de601 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 65c17200-3d30-4eb5-979a-5e95b5105a44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8003b} kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: Started container splitter-graph-8003b metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427c6680aef1 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 9274346f-1e81-4856-9cfc-4b78f2d8ede7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8003b} kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Stopping container splitter-graph-8003b metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427ea25faeb6 namespace: kserve-ci-e2e-test resourceVersion: "26390" uid: 5b0c6a79-9437-4171-bd61-f6fbe8b77492 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8003b} kind: Pod name: splitter-graph-8003b-7dc6f58dd9-9kkzl namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7ebf623a-d9ab-4322-be2d-605f666d3ac7 kind: Event lastTimestamp: "2026-07-01T19:54:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:00Z" name: splitter-graph-8003b-7dc6f58dd9-9kkzl.18be427efd859378 namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b766c164-f39f-467e-a141-0a4809bd654f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-8003b-7dc6f58dd9 namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 791b0a65-ada5-451a-b0f5-345c78423fc6 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'Created pod: splitter-graph-8003b-7dc6f58dd9-9kkzl' metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b-7dc6f58dd9.18be427c485561c9 namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: b93d93f8-ec56-4e07-a026-906eb01e2baa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-8003b namespace: kserve-ci-e2e-test resourceVersion: "26253" uid: 06be59e4-7cb7-4e6a-90fb-6e55dfcc7c6b kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: Scaled up replica set splitter-graph-8003b-7dc6f58dd9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b.18be427c475165e0 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: e89de5d2-1c18-43f4-8f1a-b8adbfe09902 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8003b namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: bc83d130-ee6e-4a48-8be8-2d92c0144ba7 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: InferenceGraph [splitter-graph-8003b] is Ready metadata: creationTimestamp: "2026-07-01T19:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:24Z" name: splitter-graph-8003b.18be427c4d25ab1e namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 5866109a-5cbc-4b35-81a1-ea5fea6e6528 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:54.047549Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-00068-predictor-cd7ff766-4rlln to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be42754c84eaf4 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: 4cee2326-09ef-4539-8a99-289645fa0797 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25920" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Add eth0 [10.132.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be427566b260bb namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: e1201ddc-50d5-4cd6-8b15-68c35a3fa0a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be42756827ae04 namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: 299c86f3-cd1d-4eea-86b1-a3d72c678fc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be42756b533304 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: bdd26dbe-8025-4b57-b781-ec637be6f851 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be42756bc49a31 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 45c4237d-8361-4a14-be96-e5b7942d70df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be42756bcde682 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: ef48b917-8ac9-43d3-ad02-2a126d5d47d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be4275703d4630 namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: ef770f41-2bc6-4e28-8156-007e3daf8f88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be427570adb98d namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 835b7714-72a6-4ee3-baea-fbfc16ed3b1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T19:53:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:31Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be4275bd9d2796 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 43ac2ee9-da73-4562-a0da-273c46b45eca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T20:10:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:13Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be43674b431a5f namespace: kserve-ci-e2e-test resourceVersion: "33620" uid: 36ccd72f-77f8-4070-aa3b-ec8fce7ecf37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-00068-predictor-cd7ff766-4rlln namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 1d7fd533-c9ad-40ec-b822-fe5037faae02 kind: Event lastTimestamp: "2026-07-01T20:10:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:13Z" name: success-200-isvc-00068-predictor-cd7ff766-4rlln.18be43674b437a87 namespace: kserve-ci-e2e-test resourceVersion: "33621" uid: a8edc92c-1b87-4fcf-843a-dfdd0ba457dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-00068-predictor-cd7ff766 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: ce144c50-0f64-4fe2-ae36-ba44534baefb kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: 'Created pod: success-200-isvc-00068-predictor-cd7ff766-4rlln' metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-00068-predictor-cd7ff766.18be42754c0c4652 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: c4600530-842b-4935-8685-36f8188011a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 0fef8590-a79d-4e97-9061-2cf1943b7c75 kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: Scaled up replica set success-200-isvc-00068-predictor-cd7ff766 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:53Z" name: success-200-isvc-00068-predictor.18be4275494e1409 namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 9b2f4811-fba6-4f9d-9940-7d53e513b4cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 23a5f314-e84a-49c2-ba73-700d783a38c0 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: success-200-isvc-00068-predictor.18be4278c83404e9 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 0763232a-702f-4863-9f42-28742c12bca4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 23a5f314-e84a-49c2-ba73-700d783a38c0 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: success-200-isvc-00068-predictor.18be4278c8a1bbcc namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 3e282d53-6706-4a97-adfc-86a6583b0bb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: 23a5f314-e84a-49c2-ba73-700d783a38c0 kind: Event lastTimestamp: "2026-07-01T19:54:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:09Z" name: success-200-isvc-00068-predictor.18be427fc60e212b namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 3d0c8422-2429-4f70-afb7-f7c506fee9d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00068-predictor namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: 23a5f314-e84a-49c2-ba73-700d783a38c0 kind: Event lastTimestamp: "2026-07-01T19:54:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:09Z" name: success-200-isvc-00068-predictor.18be427fc674346e namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 71b479b0-d6b7-48af-a0ea-a77ec332b2a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:52:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "25859" uid: 60670cd4-5fd4-4cfc-a0ff-15bc32bee4d3 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:42Z" name: success-200-isvc-00068.18be427549468d3f namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 3a95c1a7-a6ed-46a2-a732-de080150abe9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00068 namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 60670cd4-5fd4-4cfc-a0ff-15bc32bee4d3 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: InferenceService [success-200-isvc-00068] is Ready metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:42Z" name: success-200-isvc-00068.18be428078ffcd2d namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: d6a30e74-fbba-4829-a9b7-342eb7eb3932 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:59.095670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-25c73-predictor-746989f766-6kkr9 to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eac69dcb83 namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 0d77609e-d719-4ebf-905c-213aa4284f42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-25c73-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41ead0dd528a namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 70020b91-f91c-4ef9-8dc2-f34f576aa5a7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb3a13097b namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 10968be8-e944-4b63-8ea7-66e392b37a7d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb3b75bc80 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: ba6000f0-172c-4474-a7c6-742f67091272 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb3e5e2653 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: a15ae230-642e-4f4d-b35b-98a38c15c283 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb3eca96a6 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 74b4a2cc-28b6-4da9-bab9-69706c5d1b2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb3ed2f8cf namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 7d63c841-ea30-4206-b763-568932235994 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb41fffb7f namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 5ddc9065-421e-4298-8737-abf21da9a536 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:01Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41eb426dd23e namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: c26deb64-40f7-4ece-ba24-49b98b0edb02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be41ebb46a13c0 namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: b0f65a05-a405-41f7-bdcc-7c3e75ba1ce7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be426e50ba57a0 namespace: kserve-ci-e2e-test resourceVersion: "25241" uid: 36f053cc-06ca-4c20-9229-d656f1e4b12a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be426e50bb856d namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 113bc975-ca00-490e-8b9d-cbcdd336f564 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-25c73-predictor-746989f766-6kkr9 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: b8128d6a-7d30-43d4-b37b-2cb6d3256c13 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-25c73-predictor-746989f766-6kkr9.18be426e52e2df67 namespace: kserve-ci-e2e-test resourceVersion: "25252" uid: 9ec3f770-05a3-4909-8532-bc13debe8072 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-25c73-predictor-746989f766 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 855e169f-d6c6-4113-857b-a4758deef734 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Created pod: success-200-isvc-25c73-predictor-746989f766-6kkr9' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73-predictor-746989f766.18be41eac61f48f7 namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: 6a5b06bd-c888-4e66-a067-29dd462abe66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 9ae0959f-6123-4f12-b335-2b334b9a7b19 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Scaled up replica set success-200-isvc-25c73-predictor-746989f766 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73-predictor.18be41eac3bafb6a namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 5e4138f4-695b-425a-b527-8a4053a3680a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: f689eb7f-46a9-4171-84cf-4550a18ea1ea kind: Event lastTimestamp: "2026-07-01T19:43:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:44Z" name: success-200-isvc-25c73-predictor.18be41ee427668e8 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 32775594-f9f0-4573-8643-0226b31094d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: f689eb7f-46a9-4171-84cf-4550a18ea1ea kind: Event lastTimestamp: "2026-07-01T19:43:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:44Z" name: success-200-isvc-25c73-predictor.18be41ee43438d32 namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 201fb413-fcb4-408c-bc42-a5f75ec7f767 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: f689eb7f-46a9-4171-84cf-4550a18ea1ea kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:14Z" name: success-200-isvc-25c73-predictor.18be41f8bfdc3bfa namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: f2b185ff-ee11-469e-a5f0-342f00abdd71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-25c73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: f689eb7f-46a9-4171-84cf-4550a18ea1ea kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:14Z" name: success-200-isvc-25c73-predictor.18be41f8c04ea5d7 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: 0844f679-0846-40b7-bd4a-c6db23a65393 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: cd212084-2226-4a59-a330-59462bf58e70 kind: Event lastTimestamp: "2026-07-01T19:43:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:49Z" name: success-200-isvc-25c73.18be41eac3c0819c namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: 2a075035-947f-4d70-ab12-e53d513c407c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: cd212084-2226-4a59-a330-59462bf58e70 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-25c73": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-25c73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73.18be41eac48b0aec namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 5b126dc3-fcd0-435d-9511-64b726899b69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: cd212084-2226-4a59-a330-59462bf58e70 kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-25c73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-25c73.18be41eac48b5119 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: dc56cde7-1a90-4f83-af64-aafb4beec3ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-25c73 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: cd212084-2226-4a59-a330-59462bf58e70 kind: Event lastTimestamp: "2026-07-01T19:43:49Z" message: InferenceService [success-200-isvc-25c73] is Ready metadata: creationTimestamp: "2026-07-01T19:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:49Z" name: success-200-isvc-25c73.18be41f6725f8d2d namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 616f980d-cb83-4056-93cf-f4121b0bc037 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:41:48.857738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-44b6c-predictor-788df44c68-prgzl to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:41:48Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41da6c1c06e0 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: fb7603f0-7fe6-45d2-bd9c-3c4101a0ec09 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-44b6c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41da78e6691f namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: d04a7791-5488-4e0b-a479-11dda259e053 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41daa999bcd1 namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 0adc4cc1-33f0-46d4-9962-8cc29b3ce0b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41daaaee4b13 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 035e58b6-4661-493a-ab70-198212dcec12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41daae71cb22 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 3034884d-7df4-47a7-ac54-d1c8ee8cceb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41daaee0f939 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 25890c83-d26d-40ed-a5db-0749a81119b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:49Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41daaeea4737 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 64db11d0-a8be-4c8a-ab91-0b47129bbf5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:50Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41dab1f55aa3 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 9fcdb633-fdc9-482f-9724-47bb5d86c508 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:41:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:50Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41dab264c981 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: a7d42bf7-5c14-45e6-a895-4209b3ec8da6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:42:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:27Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41db1540f482 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: cb66f4bc-d218-4ef5-b973-82316c1e8c5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41eac54f786d namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: f4992535-ef59-4103-b4da-17bb472a2397 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:42:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:59Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41eac5502356 namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 8ad328b1-e0b8-4c48-bccc-a0ea0a60694b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b6c-predictor-788df44c68-prgzl namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: c217f2fc-cd4e-4286-a5fd-961a1809ca4f kind: Event lastTimestamp: "2026-07-01T19:43:02Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:02Z" name: success-200-isvc-44b6c-predictor-788df44c68-prgzl.18be41eb9d8c1762 namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 2e83a43b-ff8a-4694-85e8-b4725fb6aa8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-44b6c-predictor-788df44c68 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 4a1686f7-a929-4d4f-a7f9-c359243955d6 kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: 'Created pod: success-200-isvc-44b6c-predictor-788df44c68-prgzl' metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:41:48Z" name: success-200-isvc-44b6c-predictor-788df44c68.18be41da6bb85e06 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 62ad0644-787e-4a23-943e-31cc05703238 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 178e8758-73c0-41bb-a84c-5f4614a61091 kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: Scaled up replica set success-200-isvc-44b6c-predictor-788df44c68 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:41:48Z" name: success-200-isvc-44b6c-predictor.18be41da67eff796 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 6b4f8e2b-5613-4fe4-bd0c-bc7de540f781 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 35151055-cdb8-4973-8652-20706369aff9 kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:18Z" name: success-200-isvc-44b6c-predictor.18be41dde6b9b37b namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: dedf5806-ba87-4e93-a521-bbec2bdfad8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 35151055-cdb8-4973-8652-20706369aff9 kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:18Z" name: success-200-isvc-44b6c-predictor.18be41dde71b3a44 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: b441a24b-ff1f-413e-a369-9e7493df4ab3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 35151055-cdb8-4973-8652-20706369aff9 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: success-200-isvc-44b6c-predictor.18be41e4e50352f4 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 240802ab-3fcc-41cc-9bbb-14b1fb36c00b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 35151055-cdb8-4973-8652-20706369aff9 kind: Event lastTimestamp: "2026-07-01T19:42:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:48Z" name: success-200-isvc-44b6c-predictor.18be41e4e57705b2 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: fcda4e59-e667-40dd-9137-77d77fb55568 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44b6c namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: b8fc8977-361c-4b5d-88ef-9f312c9518e6 kind: Event lastTimestamp: "2026-07-01T19:42:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:37Z" name: success-200-isvc-44b6c.18be41da68038326 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 62b72fb1-9b30-4410-9504-b1412551b52d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44b6c namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: b8fc8977-361c-4b5d-88ef-9f312c9518e6 kind: Event lastTimestamp: "2026-07-01T19:42:37Z" message: InferenceService [success-200-isvc-44b6c] is Ready metadata: creationTimestamp: "2026-07-01T19:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:37Z" name: success-200-isvc-44b6c.18be41e5d02c8f63 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: c93372a5-8a3c-49e9-a0dc-9abb5d4fd883 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:34.278242Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427eaa72adc4 namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: 7da1bb5a-3c80-4b3e-8af3-2125330cd5cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Add eth0 [10.132.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ec48edfe8 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: 98fc11b4-a9a9-4838-b3c9-25ceca70e542 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ec5f16c24 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: e124214d-79e5-4a88-ae76-36a47a660bab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ec9697373 namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 902f4221-a31e-417e-9b4f-1f5481411f0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ec9f9bc2b namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 526809a2-d712-499f-b487-85998b37d5ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427eca02c26a namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: 82915533-3491-4f24-a9ec-1d510afcc6b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ecdd263a4 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: a6184566-4b96-4982-8a67-17fe55d8f007 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427ece40fde1 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 27325be0-4e17-468e-8029-9540a0d662be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T20:02:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:51Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be427edee49f9f namespace: kserve-ci-e2e-test resourceVersion: "30581" uid: 8d6b6e4e-52d0-456f-b203-0e0550f22cd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T20:02:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:49Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be42ffdf0be2ce namespace: kserve-ci-e2e-test resourceVersion: "30544" uid: 71c03c55-ade2-412e-92eb-55ef8d800273 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T20:02:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:49Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be42ffdf0cc8a7 namespace: kserve-ci-e2e-test resourceVersion: "30545" uid: 14bb5101-17b2-44cf-b7c4-4a946825f478 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:02:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3031d17b-7f58-4c88-b495-1e08101ac08f kind: Event lastTimestamp: "2026-07-01T20:02:51Z" message: 'Readiness probe failed: Get "https://10.132.0.54:8643/healthz": dial tcp 10.132.0.54:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:02:51Z" name: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x.18be430053aa9d4c namespace: kserve-ci-e2e-test resourceVersion: "30580" uid: 1f5ef349-bf65-4b83-91a8-bd8d237e2b21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5e51f-predictor-6bcc4458db namespace: kserve-ci-e2e-test resourceVersion: "26409" uid: b09bbea6-7b21-48f4-ae46-09e241cc5943 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Created pod: success-200-isvc-5e51f-predictor-6bcc4458db-cbk6x' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor-6bcc4458db.18be427eaa09e4cf namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: b9af526f-c28d-4ea4-8472-0fec9b37affd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: c17625a7-cef5-41b7-ba07-014dba472f4d kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Scaled up replica set success-200-isvc-5e51f-predictor-6bcc4458db from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f-predictor.18be427ea81a04ba namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 4db4861d-b06e-4f47-a7cf-6053dd8ad164 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 8d42661a-32f8-4ad3-81f8-e0afaf5af90e kind: Event lastTimestamp: "2026-07-01T19:53:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:49Z" name: success-200-isvc-5e51f-predictor.18be4282277586ba namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 8016387a-5e18-48bf-96dc-d839977911b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 8d42661a-32f8-4ad3-81f8-e0afaf5af90e kind: Event lastTimestamp: "2026-07-01T19:53:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:49Z" name: success-200-isvc-5e51f-predictor.18be4282280d5a11 namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 5abc7814-ade7-4444-9419-ccc4705bf0b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:54:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 8d42661a-32f8-4ad3-81f8-e0afaf5af90e kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: success-200-isvc-5e51f-predictor.18be4285a70a0e3b namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 8f29ad07-83ef-4031-8d1f-4dbe011ca080 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:54:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e51f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 8d42661a-32f8-4ad3-81f8-e0afaf5af90e kind: Event lastTimestamp: "2026-07-01T19:54:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:49Z" name: success-200-isvc-5e51f-predictor.18be4285a777dc06 namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: 2fdc9bfc-5e1a-4960-9fa8-5e7e08d9dbd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 2ad478ee-84ea-4a71-a332-3e53aa388da9 kind: Event lastTimestamp: "2026-07-01T19:54:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:21Z" name: success-200-isvc-5e51f.18be427ea874db37 namespace: kserve-ci-e2e-test resourceVersion: "26973" uid: 4eb7bd9c-cf0b-4683-b30e-62afe66bf88a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 2ad478ee-84ea-4a71-a332-3e53aa388da9 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-5e51f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5e51f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f.18be427ea958b23d namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 1532b42c-1a63-4db5-be38-05343e38de6c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 2ad478ee-84ea-4a71-a332-3e53aa388da9 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5e51f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-5e51f.18be427ea958fd6e namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 6e20b094-00e9-4bfb-bd27-142e7a338db7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e51f namespace: kserve-ci-e2e-test resourceVersion: "26971" uid: 2ad478ee-84ea-4a71-a332-3e53aa388da9 kind: Event lastTimestamp: "2026-07-01T19:54:21Z" message: InferenceService [success-200-isvc-5e51f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:21Z" name: success-200-isvc-5e51f.18be42899a1735cd namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: 4bb71cba-cccf-4c87-9919-e4c91df337b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:32:33.876229Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:32:33Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be415934a2415c namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: bcb5ce2e-073c-4808-bd4d-1f4a4e9b9739 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6c6c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41593fc5adf4 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: a8b9af93-0220-439a-a931-7664e576bed3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be4159723989bb namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 69fe1360-2a2b-48cb-b01e-457e7a4cad1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41597387d652 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: d888c0dd-f434-4bc6-bd81-7e6f53869f9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be415976bba508 namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: b5c9f87c-e7dd-4b9d-9407-0174d2ff6ac2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41597731f956 namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: d624819d-cf60-47e7-84d7-80e0f6aacfad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be4159773bb1be namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 9bdd64ac-aaa7-45b8-b1e9-0950629aac71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:35Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41597a677796 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 4cc8f521-1e9c-489c-909a-f8f6877ca784 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:32:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:35Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41597af89adc namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 9a058d19-8178-4d88-b688-ef363162161e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:33:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:12Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be4159b3584770 namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 2f368504-ca94-4c3b-a561-92ae214b4ae0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:48Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41da6bbda1b8 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: 7a4156cb-cbda-44b8-96ef-634f35c1e461 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 00028cdb-adec-4d60-a662-dd3dfa6fba7e kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:48Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl.18be41da6bc4a018 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 8b34e2d4-2199-4f92-af7d-16caf3c6d457 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6c6c1-predictor-6c95f7b7dd namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 1181bed6-b2e2-476c-941a-a4f7f312e6ed kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: 'Created pod: success-200-isvc-6c6c1-predictor-6c95f7b7dd-wx4sl' metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:33Z" name: success-200-isvc-6c6c1-predictor-6c95f7b7dd.18be415934030ea8 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: ff43e30e-c754-4324-840f-15a1002c4005 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 2eb88aba-66fe-425b-a2fa-6c45de5f13a4 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Scaled up replica set success-200-isvc-6c6c1-predictor-6c95f7b7dd from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:33Z" name: success-200-isvc-6c6c1-predictor.18be415932327f78 namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: 70be69b3-9240-434b-acd0-a92d3fc61d22 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 303688fe-f69b-4198-b3ad-b4a1bc74d9bf kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: success-200-isvc-6c6c1-predictor.18be415cb14f811a namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 12281b40-fd4f-4606-b2ab-0de932205ae9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 303688fe-f69b-4198-b3ad-b4a1bc74d9bf kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:03Z" name: success-200-isvc-6c6c1-predictor.18be415cb19e25fa namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 3e399ee7-cb7f-4b13-b9b7-58610e3bb76e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 303688fe-f69b-4198-b3ad-b4a1bc74d9bf kind: Event lastTimestamp: "2026-07-01T19:33:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:48Z" name: success-200-isvc-6c6c1-predictor.18be4163aeeab813 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 562fce77-53e2-4dd5-8e68-a528b8a110b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c6c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 303688fe-f69b-4198-b3ad-b4a1bc74d9bf kind: Event lastTimestamp: "2026-07-01T19:33:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:48Z" name: success-200-isvc-6c6c1-predictor.18be4163af54331f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: c6fa2792-ff68-4b90-b7a8-3e633124fcc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 2eaaecbc-4556-4648-adbf-7ccced78f94e kind: Event lastTimestamp: "2026-07-01T19:33:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:22Z" name: success-200-isvc-6c6c1.18be41593267ab49 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 37acd244-f051-4b69-b13d-c6b8b7857b3b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 2eaaecbc-4556-4648-adbf-7ccced78f94e kind: Event lastTimestamp: "2026-07-01T19:33:22Z" message: InferenceService [success-200-isvc-6c6c1] is Ready metadata: creationTimestamp: "2026-07-01T19:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:22Z" name: success-200-isvc-6c6c1.18be41646e4919a1 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: e4b36c00-f3f0-4d24-85a8-92e6576e31d9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:03.859966Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-79b53-predictor-64ff7f459d-xjf2s to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:03Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41443f3f0e1e namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: 07f16980-a6f4-4b3a-ab3c-0dd1b9eff757 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:04Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be414458f035a7 namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 5bd2bfe9-fd33-437b-9e32-31193c3b7c04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:04Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" metadata: creationTimestamp: "2026-07-01T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:04Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41445a7ddec0 namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 06554ec1-18c9-47b5-9b8d-1b448d9c9f6c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" in 16.088s (16.088s including waiting). Image size: 1335542385 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4148197298c5 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 19a99304-5c59-4f8a-980f-dffeb08d4ceb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41481dd1d40b namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 1545f802-33aa-4e4e-a6ce-72f0e733d85d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41481e5424d6 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: bd4784c9-7515-4e40-8654-c7bfde728128 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:20Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41481e605124 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 29caca52-e041-4edc-9107-cb1a03e9cab0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.552s (2.552s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4148b681547a namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 2405ca9f-e6d1-4a5e-8820-8254f53d9548 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4148ba4f0ed3 namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 0ae18a2c-a008-45c9-8d3d-ed689ea86f42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:23Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4148bac27b4a namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 2845e0c9-ed2c-4c00-9e61-b6d5a1dc786b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:31:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:59Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4148dbddd258 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 9f11b59d-ef8c-43a9-b3ac-2a0172ad533e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:33Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be4159346297ba namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: c9e93a0c-fe56-4924-a2ed-7538ace5c491 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:33Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41593462ffff namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: ddef083d-8931-4308-84e7-78c7c59fd974 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 75bc0071-6882-493b-80ab-4ac0b0c2f01c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s.18be41596405f5c0 namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: 639f25ae-8ee2-42eb-8030-f4ebcf485532 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-79b53-predictor-64ff7f459d namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 0c34c7ec-7a44-418b-91cd-cde7d29ddaf8 kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: 'Created pod: success-200-isvc-79b53-predictor-64ff7f459d-xjf2s' metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: success-200-isvc-79b53-predictor-64ff7f459d.18be41443ed8dbff namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 2f0f7a62-d676-4730-b1d0-65a2852c95e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: d6bce020-63f7-4b1c-9ecd-04ab4fb2dc87 kind: Event lastTimestamp: "2026-07-01T19:31:03Z" message: Scaled up replica set success-200-isvc-79b53-predictor-64ff7f459d from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:03Z" name: success-200-isvc-79b53-predictor.18be41443bf57bb4 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 2bf1435d-e292-4904-a3b0-dc91c01a23fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 6a9583dd-b3c6-4764-a535-92831b0c5938 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:33Z" name: success-200-isvc-79b53-predictor.18be4147ba731068 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 55434943-c7af-461a-8510-f54c0bda13f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 6a9583dd-b3c6-4764-a535-92831b0c5938 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:33Z" name: success-200-isvc-79b53-predictor.18be4147bad6c35a namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 42d44e04-078c-4672-8380-07fbb4b31098 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 6a9583dd-b3c6-4764-a535-92831b0c5938 kind: Event lastTimestamp: "2026-07-01T19:32:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:18Z" name: success-200-isvc-79b53-predictor.18be414eb831b742 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: e2e43ff9-e44f-4af2-a98d-9fc7c3a13a68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-79b53-predictor namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 6a9583dd-b3c6-4764-a535-92831b0c5938 kind: Event lastTimestamp: "2026-07-01T19:32:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:18Z" name: success-200-isvc-79b53-predictor.18be414eb8925254 namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: bc6256b2-3f63-4cd0-83be-4cbf5018c511 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-79b53 namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: a4728b54-91e1-496e-b8fc-44f0e2f2f01a kind: Event lastTimestamp: "2026-07-01T19:32:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:09Z" name: success-200-isvc-79b53.18be41443bf82606 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 39f85376-26f6-4b1a-af75-d1cc33ee83a9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-79b53 namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: a4728b54-91e1-496e-b8fc-44f0e2f2f01a kind: Event lastTimestamp: "2026-07-01T19:32:09Z" message: InferenceService [success-200-isvc-79b53] is Ready metadata: creationTimestamp: "2026-07-01T19:32:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:09Z" name: success-200-isvc-79b53.18be41539690d9d5 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: ff662f2c-4aa6-4875-b29f-20e6c13f618f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:24.040168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25223" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8003b-predictor-7d597dc584-hxm4g to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e4feeb26e namespace: kserve-ci-e2e-test resourceVersion: "25237" uid: c01c2d5a-315a-488f-82d7-99c1b9a6e586 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8003b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e59c3d924 namespace: kserve-ci-e2e-test resourceVersion: "25286" uid: bae5109b-6b8d-4695-8a8c-ff8c38f30466 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e8e732dd5 namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: 21e33d87-cc11-4f51-a1c3-731a88e55ba6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e8fd5a568 namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: 01fc871a-9810-49f8-bfc0-b7374ebf1329 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e9354c214 namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: e566492d-b0e1-4709-9da2-d95ec0238e66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e93c80318 namespace: kserve-ci-e2e-test resourceVersion: "25316" uid: b789625e-9bab-4840-a4f0-ace400e7946f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e93d1cd3c namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: bce1ca71-523e-428c-ba56-59894e4e4e40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e9759f6d7 namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: a5d86a5e-cc0d-451a-84b3-67d616db171c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:52:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:25Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426e97dbeb16 namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: e61bf60c-bd79-4dd7-87aa-a8cb65304eb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:53:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:02Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be426ef38455b3 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 87185068-f7d9-474a-91dc-b096f9a323a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be427ea9dd5e87 namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 947a2cd3-a807-4994-b8b1-dd93b2f58771 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: a57e3829-7446-4fe1-865f-6b6745092bb7 kind: Event lastTimestamp: "2026-07-01T19:53:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:34Z" name: success-200-isvc-8003b-predictor-7d597dc584-hxm4g.18be427ea9ddf9cc namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: cd69b114-ac8a-4767-9d08-20f9e47e6dee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8003b-predictor-7d597dc584 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: 9db12a52-01ae-4d0c-8ff8-ee6f25bf4e97 kind: Event lastTimestamp: "2026-07-01T19:52:24Z" message: 'Created pod: success-200-isvc-8003b-predictor-7d597dc584-hxm4g' metadata: creationTimestamp: "2026-07-01T19:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:24Z" name: success-200-isvc-8003b-predictor-7d597dc584.18be426e4e93fa78 namespace: kserve-ci-e2e-test resourceVersion: "25231" uid: 5fa9a50d-29ba-4c43-a61a-c4275f84e066 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25206" uid: f7aa048a-0991-457f-a1a0-1c2be4af6609 kind: Event lastTimestamp: "2026-07-01T19:52:23Z" message: Scaled up replica set success-200-isvc-8003b-predictor-7d597dc584 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:23Z" name: success-200-isvc-8003b-predictor.18be426e4abe8345 namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: ad9d4907-374d-409e-87da-2c25c9823236 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25207" uid: 71437355-46d1-42bf-9842-9d71a75841e1 kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:53Z" name: success-200-isvc-8003b-predictor.18be4271c97a95a5 namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: 6d7b3dc9-0a42-420b-a56c-d9cccf24688c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25207" uid: 71437355-46d1-42bf-9842-9d71a75841e1 kind: Event lastTimestamp: "2026-07-01T19:52:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-8003b-predictor.18be4271c9d9717a namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: cd67f16d-4385-4d85-a4ec-c674477451dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 71437355-46d1-42bf-9842-9d71a75841e1 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: success-200-isvc-8003b-predictor.18be4278c833dc5c namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 0bebb883-5543-441c-b0b6-9e86e0e857cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:53:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8003b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 71437355-46d1-42bf-9842-9d71a75841e1 kind: Event lastTimestamp: "2026-07-01T19:53:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:24Z" name: success-200-isvc-8003b-predictor.18be4278c8cc67f2 namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: f7930d86-f8bb-4f8a-a39e-ea52e0bb696a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25195" uid: 0dc475cc-12df-4230-b6ca-f63869f5ffe5 kind: Event lastTimestamp: "2026-07-01T19:53:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:12Z" name: success-200-isvc-8003b.18be426e4ac215b8 namespace: kserve-ci-e2e-test resourceVersion: "26162" uid: dc1cab27-2a12-4c62-bf5e-dbc10b5fd318 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25195" uid: 0dc475cc-12df-4230-b6ca-f63869f5ffe5 kind: Event lastTimestamp: "2026-07-01T19:52:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-8003b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8003b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:23Z" name: success-200-isvc-8003b.18be426e4b5dd611 namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: b1e0cb8b-106b-4338-b211-f71086bd33da reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "25195" uid: 0dc475cc-12df-4230-b6ca-f63869f5ffe5 kind: Event lastTimestamp: "2026-07-01T19:52:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8003b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:23Z" name: success-200-isvc-8003b.18be426e4b5e1a25 namespace: kserve-ci-e2e-test resourceVersion: "25213" uid: e0fcef9b-1916-411b-a504-e8aaf020274a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8003b namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 0dc475cc-12df-4230-b6ca-f63869f5ffe5 kind: Event lastTimestamp: "2026-07-01T19:53:12Z" message: InferenceService [success-200-isvc-8003b] is Ready metadata: creationTimestamp: "2026-07-01T19:53:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:12Z" name: success-200-isvc-8003b.18be4279aeb3ee56 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 1cc391b4-dc1e-4ceb-8148-211a55a9a6e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:13.916767Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:13Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162873c69dc namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 1df0c2f4-60ab-40ee-bbfc-734f834fe409 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162a1cfaa47 namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 8cd62d46-d434-403f-ba8a-46c91ae25f68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162a33ae3b9 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: ecec22bd-44e5-4936-afde-7c7ff8c3e7de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162a66aeeda namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: ece76903-fbf4-4f63-bb79-403592210e16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162a6e59bf3 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: d6d3c3b1-0ddc-410a-bcc8-cd2099ba7ec8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162a6f22cd8 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 5b009a21-6399-426b-8f8a-08a0fb5008b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162aa1904f2 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 8942bc1a-a26d-4031-851a-36576549747e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:33:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:14Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162aaa2ffeb namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 0b3d5f3c-c65e-4589-900b-7777150077d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:42:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:31Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be4162d3e381fc namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 361a3460-1bc6-4586-b450-73feaedbfba3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be41e3b9515793 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 1f4039e8-4aac-42a3-b5de-1983ab79c5b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be41e3b951db98 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 7dea14ca-54ca-4021-a635-6351777c0017 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: aa46fc9c-d916-4629-9fce-f865f714f516 kind: Event lastTimestamp: "2026-07-01T19:42:31Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:31Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b.18be41e4487aea0f namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: f51ab710-b8d5-43d4-bc8b-1ffdf3304fe6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d109c-predictor-6dc7dbbb44 namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: d64965e3-c961-4bfc-9f09-8efc8b03f8b6 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: 'Created pod: success-200-isvc-d109c-predictor-6dc7dbbb44-vgv9b' metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:13Z" name: success-200-isvc-d109c-predictor-6dc7dbbb44.18be416286ca984d namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 90deb5a7-c799-4b4c-a77a-928fd2bae6ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 93e01fdf-a1bc-4585-8544-de357bf9903e kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Scaled up replica set success-200-isvc-d109c-predictor-6dc7dbbb44 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:13Z" name: success-200-isvc-d109c-predictor.18be41628519da79 namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 1f87c5cc-a189-4642-aaa2-b762eac52d13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 0054304d-bc9e-4ff8-a435-34f4dbda193f kind: Event lastTimestamp: "2026-07-01T19:33:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:43Z" name: success-200-isvc-d109c-predictor.18be416603971e48 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 04bd2293-d49a-47b4-9074-f7c1f40bf236 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 0054304d-bc9e-4ff8-a435-34f4dbda193f kind: Event lastTimestamp: "2026-07-01T19:33:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:43Z" name: success-200-isvc-d109c-predictor.18be416603f8cccb namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 75f4d496-6cc8-4933-824f-767f9a04a973 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 0054304d-bc9e-4ff8-a435-34f4dbda193f kind: Event lastTimestamp: "2026-07-01T19:34:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:28Z" name: success-200-isvc-d109c-predictor.18be416d020ae9ba namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 9991c718-d211-4799-b309-d818f3d279bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d109c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 0054304d-bc9e-4ff8-a435-34f4dbda193f kind: Event lastTimestamp: "2026-07-01T19:34:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:28Z" name: success-200-isvc-d109c-predictor.18be416d0267aa54 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: fc7a7e1f-403b-4139-b7b4-03bed93509e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d109c namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 9f67d610-03cf-4b95-8aed-0f15f07b2156 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-d109c.18be4162851a2473 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 66ead1d4-143c-46fb-838d-187b14d1b0b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d109c namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 9f67d610-03cf-4b95-8aed-0f15f07b2156 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: InferenceService [success-200-isvc-d109c] is Ready metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-d109c.18be416d901506cb namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: 16af83a2-d91e-4e12-a01c-445070d550dc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:39.111258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f417bb2d30 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 8d93d9e7-7bd5-4115-9373-c05e5ad6811b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f431d78662 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 2facf00b-dcfc-4b7a-b1a0-00f7ee802e8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f4335c6d7c namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: ee181751-0492-432e-a6f2-bcb788cf5556 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f4371e4573 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 7e160ff0-984c-44e0-8cb7-71e731514e47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f437d5a09c namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 143add3c-039a-4cbd-a335-65cd95218f6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f437e2b99c namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: c9ca2b41-40d0-48b2-9153-96541ad8f77d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f43c147b8d namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: e8f62aaf-b8e8-4651-939a-5faea7410692 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f43c89f988 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 1734b820-a6e7-4d6b-b33c-5460f309bd68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:52:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:57Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be41f4990ffaca namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 03b98025-9e82-4e8f-ac71-71a8340ff672 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be42754bd680c6 namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 1edc9da5-d95c-44fa-a053-3eb993cbe220 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:52:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:54Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be42754bd71bf4 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 33cda750-8e8b-4cd0-865c-658f3ff6f0a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3a79a16d-57d1-4d3c-ae44-fa833c56ef89 kind: Event lastTimestamp: "2026-07-01T19:52:57Z" message: 'Readiness probe failed: Get "https://10.132.0.48:8643/healthz": dial tcp 10.132.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:57Z" name: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf.18be42760d716c1b namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: d5a11e67-d93a-4851-ba82-9e91ef3773e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f705b-predictor-7759fdc7c4 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: e513b880-5644-436b-bee6-e3fcf97da153 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: 'Created pod: success-200-isvc-f705b-predictor-7759fdc7c4-5jkhf' metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor-7759fdc7c4.18be41f417029a6d namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: e199d4cf-b974-45ad-acfa-04a6ffd2c939 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 83e560f1-de8b-4d1d-9f99-9517ba25d8b7 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Scaled up replica set success-200-isvc-f705b-predictor-7759fdc7c4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f705b-predictor.18be41f4148fe21e namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 763a52a3-207d-4db2-9643-7dc756c877db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: f501c45e-5ac7-421d-8a26-66675faf43c4 kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: success-200-isvc-f705b-predictor.18be41f793278265 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: cf19c02d-beb0-4aa5-a31c-dcbe29339d08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: f501c45e-5ac7-421d-8a26-66675faf43c4 kind: Event lastTimestamp: "2026-07-01T19:44:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:39Z" name: success-200-isvc-f705b-predictor.18be41f7936bb932 namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: e024e98f-90ec-44ee-bf1b-6e3f1b86f103 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: f501c45e-5ac7-421d-8a26-66675faf43c4 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: success-200-isvc-f705b-predictor.18be42058fce3c3b namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: 17f838a1-9534-4f5a-92c0-0602d6af6cb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f705b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: f501c45e-5ac7-421d-8a26-66675faf43c4 kind: Event lastTimestamp: "2026-07-01T19:44:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:54Z" name: success-200-isvc-f705b-predictor.18be4205901be87f namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: bcf37744-c7a1-4191-b112-5297472f1195 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 2483bd84-64a5-4cb2-a894-b14068fe8489 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:27Z" name: success-200-isvc-f705b.18be41f414c13c6d namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: 39c6b2f7-51e4-4428-a93a-1c9fccb30675 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f705b namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: 2483bd84-64a5-4cb2-a894-b14068fe8489 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: InferenceService [success-200-isvc-f705b] is Ready metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:27Z" name: success-200-isvc-f705b.18be41ff545f6708 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6cc918ad-8e75-4efa-854e-071e0ddae7f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:28.819782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7f91-predictor-5974f76cdd-qs7px to ip-10-0-130-234.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3ba086a46 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 0bb43713-6b34-4ed5-a3f8-78584ff1a6d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3d4340963 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 995b7501-2730-4d98-8ec8-d8e8412cf0e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1693" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3d595ac27 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 14ee3e22-0153-4d3c-8689-70db828b9fcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3d8bf602c namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: 3f64e962-6368-4101-9ec2-538d7ef6f75e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3d970e07c namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: dadab117-43d3-4f94-8af1-17c3b20ae10d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3d9800788 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 4cc5ed9a-29be-4815-a066-66a66a8be9a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3dc51da0d namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 7f2b4ac3-dc9b-4e1a-995a-d22f16c4cfd5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:42:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:29Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e3dcbda343 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: b62982b9-90fa-41fe-a9ae-8af4f891bd7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:43:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:06Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41e4363b342f namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 2bc75ea1-0c9f-498a-8b6d-6a0b6ccbce1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41f41790e6e9 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 99619ac3-2343-47ef-ba6b-b06336b33a0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:43:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:39Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41f417915357 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 5a14730f-3147-4ada-aedc-32095368aa91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 334395e0-b09e-41b9-bb5c-4eb4b371eaa8 kind: Event lastTimestamp: "2026-07-01T19:43:41Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:41Z" name: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px.18be41f4be9c1bcd namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: d82b53d6-f48e-4d81-ade0-d105f055fb92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-234.ec2.internal source: component: kubelet host: ip-10-0-130-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7f91-predictor-5974f76cdd namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: be5954e8-467e-4158-ac0b-e54053551102 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'Created pod: success-200-isvc-f7f91-predictor-5974f76cdd-qs7px' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-f7f91-predictor-5974f76cdd.18be41e3b97e9108 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: fa13ef6b-6658-4ed3-9a92-50c5d9343084 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 45f19cbb-2e7a-4ba4-8e90-9d2124526656 kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: Scaled up replica set success-200-isvc-f7f91-predictor-5974f76cdd from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-f7f91-predictor.18be41e3b77a7e89 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: e3da0db9-7a65-4fa3-ab14-fe9f95d71b79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 86049bf6-eded-45cd-9b1f-ff0f38c33ef9 kind: Event lastTimestamp: "2026-07-01T19:42:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:58Z" name: success-200-isvc-f7f91-predictor.18be41e735ef177f namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: fc12223c-c5d1-4c22-b628-4e4f087ddd7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 86049bf6-eded-45cd-9b1f-ff0f38c33ef9 kind: Event lastTimestamp: "2026-07-01T19:42:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:58Z" name: success-200-isvc-f7f91-predictor.18be41e7365531ad namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 82187ce1-a212-4987-9c11-49b2c79378dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 86049bf6-eded-45cd-9b1f-ff0f38c33ef9 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: success-200-isvc-f7f91-predictor.18be41ee34ac0148 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: fed3e08d-1cec-4365-ba9c-2ccb8f67ed89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 86049bf6-eded-45cd-9b1f-ff0f38c33ef9 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:28Z" name: success-200-isvc-f7f91-predictor.18be41ee3505a31d namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 0e26e75f-dc62-472a-9908-a3dc3c8d6af1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: 277667f4-95de-4560-8461-28abbceb4ead kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-f7f91.18be41e3b7a92655 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 396413fc-8418-4e14-ad99-8d740929f3b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: 277667f4-95de-4560-8461-28abbceb4ead kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-f7f91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-f7f91.18be41e3b82d9841 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 7819ea14-e655-441d-b0ac-611534249ee6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: 277667f4-95de-4560-8461-28abbceb4ead kind: Event lastTimestamp: "2026-07-01T19:42:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:28Z" name: success-200-isvc-f7f91.18be41e3b82e2789 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 04378267-ce63-4eb1-9709-1998edd98838 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7f91 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 277667f4-95de-4560-8461-28abbceb4ead kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: InferenceService [success-200-isvc-f7f91] is Ready metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: success-200-isvc-f7f91.18be41eef19080ec namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: d16aa014-955f-4543-a9bb-995c38bc35ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:54.090392Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-00068-5f66c94d7d-sd6wz to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be4283475810d2 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: 139128f7-123c-40d3-9b92-76e84a5e7267 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be428360a86b67 namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 426a529e-634a-439e-9907-c1ab364678d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-00068} kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be428362078f10 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: b7867b4a-5416-44b8-a548-fb85026e8c4e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-00068} kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: 'Created container: switch-graph-00068' metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be4283650fac84 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 9a140aa2-70e0-4b3c-930c-f53fe2d00311 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-00068} kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: Started container switch-graph-00068 metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be4283657f17be namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: cdce835e-67a5-473d-8f95-e2a67360f8c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-00068} kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T20:10:13Z" message: Stopping container switch-graph-00068 metadata: creationTimestamp: "2026-07-01T20:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:13Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be43674186373e namespace: kserve-ci-e2e-test resourceVersion: "33601" uid: c9595953-faa1-4f82-9cb5-257c5016ff7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-00068} kind: Pod name: switch-graph-00068-5f66c94d7d-sd6wz namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: ad84eab5-b20e-4b2d-829e-11623caf0a1d kind: Event lastTimestamp: "2026-07-01T20:10:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:10:40Z" name: switch-graph-00068-5f66c94d7d-sd6wz.18be4367d8976f30 namespace: kserve-ci-e2e-test resourceVersion: "33874" uid: bdf5c9ad-7047-43e6-bff1-91df2d2559fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-00068-5f66c94d7d namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: f2c38fe4-f245-462c-867a-ed5c64aafdd9 kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: 'Created pod: switch-graph-00068-5f66c94d7d-sd6wz' metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068-5f66c94d7d.18be428346eb61c1 namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: 77c90971-c496-48b2-9246-6e14b6356e4e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-00068 namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: 01664269-b808-44b8-ad86-43addf20d636 kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: Scaled up replica set switch-graph-00068-5f66c94d7d from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068.18be428345e9d9fc namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: ef040e13-88fc-4e26-b8a5-a9fb68e5c97c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-00068 namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: ff021ea1-ccc1-408e-b554-4bb8e0281bf9 kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: InferenceGraph [switch-graph-00068] is Ready metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:54Z" name: switch-graph-00068.18be42834a0f3877 namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: 635efbe9-8dea-42a5-a735-9ac289dd98f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-00068 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: f63f5ab5-4b7b-4bcb-a8e3-a8e0b2956113 kind: Event lastTimestamp: "2026-07-01T19:54:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:24Z" name: switch-graph-00068.18be4286c485a4f0 namespace: kserve-ci-e2e-test resourceVersion: "27001" uid: 281296b0-5103-4aed-bd44-d810bf08e530 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-00068 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: f63f5ab5-4b7b-4bcb-a8e3-a8e0b2956113 kind: Event lastTimestamp: "2026-07-01T19:54:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:24Z" name: switch-graph-00068.18be4286c4e766c8 namespace: kserve-ci-e2e-test resourceVersion: "27002" uid: b716214d-dd8c-4d5a-bb07-968231220d8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:33.937738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6c6c1-578d4456d5-mfhwn to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:33Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be416730942395 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 2b35755b-8153-4cf4-b28d-aa5b62e9f9d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:33:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-6c6c1-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:34Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be416739dc952a namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 4f06d3b2-964c-495e-9c8c-6a69f59deeb8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:33:34Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:34Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be41676d8615a9 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: f6ebcac4-7d08-4ddb-8db1-b3930b3bb4a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c6c1} kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:33:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:34Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be41676ed611bc namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 0a57d137-2e86-4466-92ae-54b467423834 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c6c1} kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:33:35Z" message: 'Created container: switch-graph-6c6c1' metadata: creationTimestamp: "2026-07-01T19:33:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:35Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be416771de5d4b namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 91fc0567-0417-41bb-9a0e-c3c64c624592 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c6c1} kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:33:35Z" message: Started container switch-graph-6c6c1 metadata: creationTimestamp: "2026-07-01T19:33:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:35Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be41677293c052 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: bcaf22a6-3ec0-4ee6-860e-b5a964270074 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c6c1} kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:41:48Z" message: Stopping container switch-graph-6c6c1 metadata: creationTimestamp: "2026-07-01T19:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:41:48Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be41da5ebe3f1c namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: fe0095aa-2e9a-4cc0-9304-72ae5e6356ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c6c1} kind: Pod name: switch-graph-6c6c1-578d4456d5-mfhwn namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 795cf749-d561-441f-ade3-9ad1ae0f5c08 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: switch-graph-6c6c1-578d4456d5-mfhwn.18be41db0f09e2f2 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 272d3482-ec6b-434b-a941-c6ee93c95206 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6c6c1-578d4456d5 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 83acb363-a489-4873-8ef6-b2ff005a7d26 kind: Event lastTimestamp: "2026-07-01T19:33:33Z" message: 'Created pod: switch-graph-6c6c1-578d4456d5-mfhwn' metadata: creationTimestamp: "2026-07-01T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:33Z" name: switch-graph-6c6c1-578d4456d5.18be416730355788 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 364e17c6-79fe-4a9b-8064-2df0ad6415c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 08ad7b4a-4e06-416b-ab85-28546f49abbe kind: Event lastTimestamp: "2026-07-01T19:33:33Z" message: Scaled up replica set switch-graph-6c6c1-578d4456d5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:33Z" name: switch-graph-6c6c1.18be41672f1269a7 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 2580c333-292c-4c85-9c98-27495e652919 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 76122add-3fdf-4027-93a4-d5852b3cd24f kind: Event lastTimestamp: "2026-07-01T19:33:34Z" message: InferenceGraph [switch-graph-6c6c1] is Ready metadata: creationTimestamp: "2026-07-01T19:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:34Z" name: switch-graph-6c6c1.18be4167354e7425 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 772daed2-5c4a-425c-b4c2-e6c8e7890a8a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 321845f1-5a6b-4d23-9093-afd15483788a kind: Event lastTimestamp: "2026-07-01T19:34:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:03Z" name: switch-graph-6c6c1.18be416aad80234f namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: b7df56fe-5577-411c-a345-1095b117521f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6c6c1 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 321845f1-5a6b-4d23-9093-afd15483788a kind: Event lastTimestamp: "2026-07-01T19:34:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:03Z" name: switch-graph-6c6c1.18be416aadd050c4 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 1ec04c16-adf9-4b6e-b1db-c551f1a50c49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:32:23.598860Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-79b53-c5dc9c995-jnsgl to ip-10-0-141-93.ec2.internal metadata: creationTimestamp: "2026-07-01T19:32:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:32:23Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be4156d00dfd26 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: ea96139a-3156-4d14-9870-76b10da9c34f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5594759958-s4lz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-79b53-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:23Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be4156d990851f namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: f5593bcf-5d1e-4794-bd5c-8bb5d1a2d864 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:24Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:32:24Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be41570cc4e820 namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: 7b332c08-e33c-471f-907c-af3ae5ad7198 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:24Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" metadata: creationTimestamp: "2026-07-01T19:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:24Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be41570e0b2d9e namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 281b40d1-2ca0-4f32-99a6-fb5ebe43d512 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:981370d72da89fef29e4477b93004a240210f608c7b11972c1e81a90201bb359" in 2.401s (2.401s including waiting). Image size: 218083966 bytes.' metadata: creationTimestamp: "2026-07-01T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:27Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be41579d2b0072 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 6c9e5cd8-b86f-4450-84f1-d37675c508fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:27Z" message: 'Created container: switch-graph-79b53' metadata: creationTimestamp: "2026-07-01T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:27Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be41579fc30a38 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 9ed0e074-63a7-4c09-be5e-7269e08ccc9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:27Z" message: Started container switch-graph-79b53 metadata: creationTimestamp: "2026-07-01T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:27Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be4157a02185bd namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: d6bbce05-3e31-48bc-86eb-26bfc1795001 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:32:33Z" message: Stopping container switch-graph-79b53 metadata: creationTimestamp: "2026-07-01T19:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:33Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be41592abd4b8e namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: c55f9888-37dc-42af-93c1-db3bbf6b8bf0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-79b53} kind: Pod name: switch-graph-79b53-c5dc9c995-jnsgl namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 44e6f124-7184-4c32-a935-23b84f003d67 kind: Event lastTimestamp: "2026-07-01T19:33:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:03Z" name: switch-graph-79b53-c5dc9c995-jnsgl.18be415a3f150bc9 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 03888240-b248-4bcb-8950-09214e56b272 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-93.ec2.internal source: component: kubelet host: ip-10-0-141-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-79b53-c5dc9c995 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 08c3fb13-cee0-4ec8-92cb-a73339a8c00e kind: Event lastTimestamp: "2026-07-01T19:32:23Z" message: 'Created pod: switch-graph-79b53-c5dc9c995-jnsgl' metadata: creationTimestamp: "2026-07-01T19:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:23Z" name: switch-graph-79b53-c5dc9c995.18be4156cf79466f namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 86ba890f-be55-4ed7-bd0a-1a6f405d11c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-79b53 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: fc05e782-e301-48e4-8d3e-bffe0f5a7f5f kind: Event lastTimestamp: "2026-07-01T19:32:23Z" message: Scaled up replica set switch-graph-79b53-c5dc9c995 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:23Z" name: switch-graph-79b53.18be4156cea29d86 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: b1d2fadb-17f8-4884-a230-f77ce608919c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-79b53 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 0c2e4767-2889-4c5e-be16-6fe7fec52412 kind: Event lastTimestamp: "2026-07-01T19:32:23Z" message: InferenceGraph [switch-graph-79b53] is Ready metadata: creationTimestamp: "2026-07-01T19:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:23Z" name: switch-graph-79b53.18be4156d44ea3db namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: ae221a41-7218-4d64-b94c-790b0fa1e29b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33874"