--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-17T13:09:21.004380Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26178" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T13:09:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T13:09:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315b8d79cc810 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 2b55060d-0aa4-4f43-bc1f-f85f6c6116a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:21Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T13:09:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315b8f181a8b5 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: b270a3d6-2228-444a-80d6-0a41a0aef9de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315b8f2e69b3e namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: acaf2e29-c9c4-4f76-86c8-9949fcb64239 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315b8f5a80b2f namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: a05a1673-faf4-4552-bb85-3316d592ec13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315b8f6210e1e namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: d6b7fd16-4c82-472d-b559-7a5de2568aa6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb11d6522c namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 84af377e-b344-4891-972d-f5f2cff93bb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb1526ced2 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 76a856da-492a-4d8c-921b-ff3adede4b61 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb159ada0c namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: 987ecc78-54e1-4a2f-b3bb-11dd1fcaa48c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb15a47b14 namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: 49d04b20-9567-452d-a93b-4ea70c65bfed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb196dabd2 namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: 643f0a22-9dcc-48f7-a690-93c1b4fd4ef3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:30Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb19e47768 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: f62c188a-4454-4262-8205-fd6f203cf4b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-17T13:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bb89a74ca0 namespace: kserve-ci-e2e-test resourceVersion: "26333" uid: 39a0192c-e661-400e-b322-28b55c099e23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:51Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bfd7a4e8ad namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 87c3fa55-fa2b-43a1-81f9-1dbe8bba4e75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: faa9678f-5bc2-479c-b6cb-02097a94b089 kind: Event lastTimestamp: "2026-07-17T13:09:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:51Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6.18c315bfd7a5c353 namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: 7bfbdf29-2704-4c84-ba4d-15954742a8cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 8a7a714e-bb51-4650-a60b-d4849b5cdb0a kind: Event lastTimestamp: "2026-07-17T13:09:20Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-jnpv6' metadata: creationTimestamp: "2026-07-17T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c315b8d7167c67 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 7b5749a0-718f-4d4a-aebf-7eae5d3ae109 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: cd29ba2b-cd2c-4899-a829-d33dc8328dde kind: Event lastTimestamp: "2026-07-17T13:09:20Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-17T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:20Z" name: isvc-autogluon-ts-v1-predictor.18c315b8d2816121 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 36bbfd35-534b-4f0b-9169-5390e5ffde36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 554774ed-d6d5-4c59-a0e0-93f17835240a kind: Event lastTimestamp: "2026-07-17T13:09:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:50Z" name: isvc-autogluon-ts-v1-predictor.18c315bc51782c53 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: e8b20b04-3206-46e9-88a0-5b2678e4ef9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 554774ed-d6d5-4c59-a0e0-93f17835240a kind: Event lastTimestamp: "2026-07-17T13:09:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:50Z" name: isvc-autogluon-ts-v1-predictor.18c315bc51e14313 namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: dbf0a5c7-21e2-49c0-b0e8-5aeba605ea6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T13:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 7c5b54f0-3255-45a9-b126-ee2d247f9b5d kind: Event lastTimestamp: "2026-07-17T13:09:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:48Z" name: isvc-autogluon-ts-v1.18c315b8d293035b namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: 27a21417-5428-4006-8063-c3041d37df4f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 7c5b54f0-3255-45a9-b126-ee2d247f9b5d kind: Event lastTimestamp: "2026-07-17T13:09:20Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:20Z" name: isvc-autogluon-ts-v1.18c315b8d3279110 namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 9a560aef-ad31-4b7b-8e26-fc62225085b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 7c5b54f0-3255-45a9-b126-ee2d247f9b5d kind: Event lastTimestamp: "2026-07-17T13:09:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:20Z" name: isvc-autogluon-ts-v1.18c315b8d327eecc namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: c2d25891-eb22-49b9-9d4c-3914de86763d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 7c5b54f0-3255-45a9-b126-ee2d247f9b5d kind: Event lastTimestamp: "2026-07-17T13:09:48Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-17T13:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:48Z" name: isvc-autogluon-ts-v1.18c315bf48e9c0dd namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 096f0e2c-eaef-4981-b324-cb6cf5cf6c69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T13:00:49.532958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31541c18a3740 namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: efcbc3bd-4023-4103-b53b-69f47033ac87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22224" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31541db70b1b3 namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: ae6af7b7-86f1-4bd9-a7fc-1af9f951aafa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31541dd0b015a namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 7874fd52-750d-46fa-8224-5cf3b685029e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:00:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T13:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:50Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31541e095bfab namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 8d563342-12c4-48aa-ac1f-ff620e77e017 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:00:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T13:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:50Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31541e116daa5 namespace: kserve-ci-e2e-test resourceVersion: "22239" uid: 0b8f5a6e-e980-40ea-b9cf-7b07fdfed048 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:01:19Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-17T13:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:01:19Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31548b613e7fb namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: aa7bcc6c-f288-43bb-95fe-bf5e1a5f399e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m0.972s (2m0.972s including waiting). Image size: 8940077339 bytes.' metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564e09d9242 namespace: kserve-ci-e2e-test resourceVersion: "23329" uid: 78c961b5-52a9-4b8a-a314-68a1c5e92ec7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564e668bb05 namespace: kserve-ci-e2e-test resourceVersion: "23331" uid: 6de11ad2-546c-41e3-99b1-5b5473e5df45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564e7b61318 namespace: kserve-ci-e2e-test resourceVersion: "23332" uid: 2de5809f-ec11-4e87-ad27-d55841fb8b46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564e7c23ede namespace: kserve-ci-e2e-test resourceVersion: "23333" uid: d317b5bf-ba14-4bba-87f8-4f7513789ea5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564ee2df04c namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: b3a3172e-0517-4d21-be43-d6710cc7d35a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:20Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c31564ef7b6bae namespace: kserve-ci-e2e-test resourceVersion: "23336" uid: b35a8006-aeb7-483c-94f4-84b4b275927f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:03:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-17T13:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:28Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c3156540fd991c namespace: kserve-ci-e2e-test resourceVersion: "23389" uid: 060bedc3-ab0a-42fc-800f-b8cce5cb4285 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c315696027e061 namespace: kserve-ci-e2e-test resourceVersion: "23497" uid: 83bc0cc8-1e19-43a9-9d48-287ea86dba3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 4f1d11c3-b0a7-4f83-b161-232673bf74f9 kind: Event lastTimestamp: "2026-07-17T13:03:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598.18c3156960287ed8 namespace: kserve-ci-e2e-test resourceVersion: "23498" uid: ffee3289-691f-43c6-b328-7aa6a3648a0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "22179" uid: f283f3c1-be35-404e-83a5-16ac1aace595 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-hb598' metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c31541c1004e92 namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 74ddf4d4-203d-4634-913f-ca328319f3c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: 65c86109-0c8f-4c81-b56a-d399b7c221e6 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1-predictor.18c31541bc4d47b5 namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: 0dab9db9-2e53-4cec-9b90-ace221b16de1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T13:01:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: d247a3bd-71d3-4f83-9ea4-50e70c000709 kind: Event lastTimestamp: "2026-07-17T13:03:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:03:34Z" name: isvc-autogluon-v1-predictor.18c315453b20e816 namespace: kserve-ci-e2e-test resourceVersion: "23433" uid: f6071030-5494-448d-85b7-ea23a197aab3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T13:01:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: d247a3bd-71d3-4f83-9ea4-50e70c000709 kind: Event lastTimestamp: "2026-07-17T13:03:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:03:34Z" name: isvc-autogluon-v1-predictor.18c315453b88c42f namespace: kserve-ci-e2e-test resourceVersion: "23434" uid: a4bb9242-2114-4efd-8253-3066605bb46d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 51789a89-931d-44b7-bd9e-7bb6743ee3a2 kind: Event lastTimestamp: "2026-07-17T13:03:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:03:38Z" name: isvc-autogluon-v1.18c31541bc7f8c7d namespace: kserve-ci-e2e-test resourceVersion: "23467" uid: 8b51f763-cba6-464b-b7b6-60e039a4d1c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 51789a89-931d-44b7-bd9e-7bb6743ee3a2 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1.18c31541bcf39f1c namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: 5e027719-2f13-41c7-a1b0-1e8211d8e88f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 51789a89-931d-44b7-bd9e-7bb6743ee3a2 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:00:49Z" name: isvc-autogluon-v1.18c31541bcf4004d namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: bf844c56-00cb-4683-827f-0db1d906947c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "23465" uid: 51789a89-931d-44b7-bd9e-7bb6743ee3a2 kind: Event lastTimestamp: "2026-07-17T13:03:38Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-17T13:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:03:38Z" name: isvc-autogluon-v1.18c3156900cecaaa namespace: kserve-ci-e2e-test resourceVersion: "23466" uid: c9f46d5a-623b-43cf-86f0-9f1bcd501004 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T13:07:35.725199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a0547beeae namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: 8cf0fec5-a036-466c-8e2b-b4b65990e315 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-noslash-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a05db607f0 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 1ea82314-dfa5-4866-bf79-55656b298307 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:36Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T13:07:36Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a0922d2c78 namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 6a72b402-3858-4e03-8b73-a22d3f8409de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:36Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a093dfb428 namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 050f5d5a-4a72-4d57-bb4b-7e2085e7dc54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:36Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a096a5380b namespace: kserve-ci-e2e-test resourceVersion: "25382" uid: 5ef5ba8b-5fa7-4bb3-9b29-ef5d431904a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:36Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a0971b397a namespace: kserve-ci-e2e-test resourceVersion: "25383" uid: 010f80a5-5c09-426c-9289-4f9e2d691b1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d0d66389 namespace: kserve-ci-e2e-test resourceVersion: "25425" uid: 2ee7af91-c73a-46ee-a7a6-490ba63140a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d4181ed0 namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: 25215d0a-c379-498d-a3d7-805f891687b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d492c1ee namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 44038841-f7d1-49c1-acd5-02934d6ee32a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d4a0507e namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 19def621-4be9-46e0-8191-75b9bb2f12e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d8965bda namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: bef9d710-0c1f-42f3-8db7-6951d504d9de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:07:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315a1d911be17 namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: 12868fae-30c6-49ed-8edc-6ab157e28fee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:09:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:15Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315b7a312685a namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: 9df2a544-c1d6-4d65-affc-940024597c7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 20c78948-621c-493c-bf8c-117cac507dc8 kind: Event lastTimestamp: "2026-07-17T13:09:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:09:15Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj.18c315b7a313b028 namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: d2b19b90-ff20-442d-bb5b-731e26f0870b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: 086a3bb3-7e4b-42e2-8f61-1975b27596c2 kind: Event lastTimestamp: "2026-07-17T13:07:35Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-mmlgj' metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c315a053eb932c namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: 1c4ea0bc-badb-4a8a-a763-c65cb89f0aa1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: 49c8d004-d4de-4131-af4e-92e955606a19 kind: Event lastTimestamp: "2026-07-17T13:07:35Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash-predictor.18c315a04eeee62b namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: 784f467e-069f-4859-85c9-2525841a193d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:07:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "25343" uid: 920a219d-7983-4bb8-b9bd-addca9cc3945 kind: Event lastTimestamp: "2026-07-17T13:08:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:08:20Z" name: isvc-autogluon-v2-noslash-predictor.18c315a3cd64df6f namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 1c121523-5d04-48b2-bb45-2ee9b0763806 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:07:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "25343" uid: 920a219d-7983-4bb8-b9bd-addca9cc3945 kind: Event lastTimestamp: "2026-07-17T13:08:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:08:20Z" name: isvc-autogluon-v2-noslash-predictor.18c315a3cdce32da namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: f919ca4c-52c0-46f7-bfd0-b20fccb052c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T13:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: 920a219d-7983-4bb8-b9bd-addca9cc3945 kind: Event lastTimestamp: "2026-07-17T13:09:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:05Z" name: isvc-autogluon-v2-noslash-predictor.18c315a74d3689fa namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: 6c052b51-db7a-408a-9462-2e348f7ba6d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T13:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: 920a219d-7983-4bb8-b9bd-addca9cc3945 kind: Event lastTimestamp: "2026-07-17T13:09:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:09:05Z" name: isvc-autogluon-v2-noslash-predictor.18c315a74dbf66ea namespace: kserve-ci-e2e-test resourceVersion: "26019" uid: 1468b2fc-2441-4a45-977b-db01b4d54440 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 5800c4c7-874f-4c9c-8ba1-0498fe7cfc8c kind: Event lastTimestamp: "2026-07-17T13:09:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:12Z" name: isvc-autogluon-v2-noslash.18c315a04eef89d8 namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 30bd175e-a4d8-42b5-80e1-bf7dd50f46f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 5800c4c7-874f-4c9c-8ba1-0498fe7cfc8c kind: Event lastTimestamp: "2026-07-17T13:07:35Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash.18c315a04f8e7a31 namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: 3c026ca8-3bdc-4d02-9adb-b9d3f9b54379 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:07:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 5800c4c7-874f-4c9c-8ba1-0498fe7cfc8c kind: Event lastTimestamp: "2026-07-17T13:07:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:07:35Z" name: isvc-autogluon-v2-noslash.18c315a04f8edcb5 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 43f3e1a2-20c2-41c0-b406-829c54a74dd3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:09:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: 5800c4c7-874f-4c9c-8ba1-0498fe7cfc8c kind: Event lastTimestamp: "2026-07-17T13:09:12Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-17T13:09:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:09:12Z" name: isvc-autogluon-v2-noslash.18c315b6c4c768d1 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: cdfa7888-2cd2-4906-8fc8-d7c1d7555216 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T13:03:45.033782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23562" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-5xb5s to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T13:03:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T13:03:45Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156a9e344d3a namespace: kserve-ci-e2e-test resourceVersion: "23566" uid: 6e0ddfad-e873-40ad-bdaf-3605e08cd12a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:03:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-17T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:45Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156aadae3665 namespace: kserve-ci-e2e-test resourceVersion: "23575" uid: 0d1a163c-4909-44e6-8200-9032cb449f14 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23572" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:03:46Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T13:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T13:03:46Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156ade2453b6 namespace: kserve-ci-e2e-test resourceVersion: "23585" uid: 5589eeff-8f2d-4c6d-9923-418687145775 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:03:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T13:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:46Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156adf922cd6 namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: 8d2672e6-d41c-47ab-b0c4-c390c10cd412 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:03:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T13:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:46Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156ae5a7aed1 namespace: kserve-ci-e2e-test resourceVersion: "23589" uid: c336257f-a6c7-4559-80de-5df695650cd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:03:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T13:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:03:46Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156ae6b22fa3 namespace: kserve-ci-e2e-test resourceVersion: "23590" uid: 7d277019-ee94-43bb-afcd-c98241e1f28b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fc6b7cedd namespace: kserve-ci-e2e-test resourceVersion: "23743" uid: 9da8fa3b-29a7-4ba2-be05-0f207f3dd1af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fc9ddcc7e namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 2652cf53-e300-4a70-93a1-180275120dfb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fca58018d namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: 89cd8f64-89ea-4592-b67f-f7eb959b5875 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fca637789 namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 24713c91-8e89-4f21-8d3d-00b9860da9bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fcd27f989 namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 01e9736d-c227-4886-85ea-f03a1a02c30b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:04:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:04:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:04:07Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3156fcd9b5e09 namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: b8a07111-3e99-45cf-8796-b268db81cc3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:05:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:45Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3158698ea5823 namespace: kserve-ci-e2e-test resourceVersion: "24449" uid: 241b68e8-7fef-4594-90e5-660a38cee4eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xb5s namespace: kserve-ci-e2e-test resourceVersion: "23564" uid: df34c9b5-36c1-47fc-958b-c3d56d9f1b6c kind: Event lastTimestamp: "2026-07-17T13:05:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:45Z" name: isvc-autogluon-v2-predictor-67b849d456-5xb5s.18c3158698eb962c namespace: kserve-ci-e2e-test resourceVersion: "24450" uid: 639d5eb3-a052-4f76-93f2-cb751dc4c9ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "23552" uid: 6dadd10e-ec0c-424b-8c5d-01c52ddeee61 kind: Event lastTimestamp: "2026-07-17T13:03:45Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-5xb5s' metadata: creationTimestamp: "2026-07-17T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:03:45Z" name: isvc-autogluon-v2-predictor-67b849d456.18c3156a9dc27130 namespace: kserve-ci-e2e-test resourceVersion: "23563" uid: 6cd4f9ca-4e16-462e-9ef8-eb0a1509b1f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23549" uid: d47c5e6d-1914-4df4-b85e-65e0d1248bf7 kind: Event lastTimestamp: "2026-07-17T13:03:44Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-17T13:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:03:44Z" name: isvc-autogluon-v2-predictor.18c3156a98dfde71 namespace: kserve-ci-e2e-test resourceVersion: "23554" uid: c63be12f-d655-4b72-83bf-d4c18e8c537a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23551" uid: 55416fec-f9e0-44dd-8cb4-6ee886aaa9c7 kind: Event lastTimestamp: "2026-07-17T13:04:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:04:30Z" name: isvc-autogluon-v2-predictor.18c3156e182b7b53 namespace: kserve-ci-e2e-test resourceVersion: "23907" uid: 560792d9-9abf-4d7e-a05a-37e1fec0d02d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23551" uid: 55416fec-f9e0-44dd-8cb4-6ee886aaa9c7 kind: Event lastTimestamp: "2026-07-17T13:04:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:04:30Z" name: isvc-autogluon-v2-predictor.18c3156e1892548b namespace: kserve-ci-e2e-test resourceVersion: "23908" uid: 17f1966b-10a1-4ae3-976b-ba2f09806cdb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-17T13:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23688" uid: 55416fec-f9e0-44dd-8cb4-6ee886aaa9c7 kind: Event lastTimestamp: "2026-07-17T13:05:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:05:45Z" name: isvc-autogluon-v2-predictor.18c315789559692d namespace: kserve-ci-e2e-test resourceVersion: "24433" uid: 24df91fc-50e9-4a1d-a6b1-80d2d09e1c4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-17T13:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "23688" uid: 55416fec-f9e0-44dd-8cb4-6ee886aaa9c7 kind: Event lastTimestamp: "2026-07-17T13:05:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:05:45Z" name: isvc-autogluon-v2-predictor.18c3157895cab29d namespace: kserve-ci-e2e-test resourceVersion: "24434" uid: db50f43e-a076-436d-a37a-6ac089f62cdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-17T13:05:50.372865Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24512" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c31587ccfef11b namespace: kserve-ci-e2e-test resourceVersion: "24516" uid: f637d6f7-24de-489c-9f75-c0257ee9972d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-variants-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c31587de85a424 namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: fc399298-be43-4ba3-943d-8aade7e54293 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24522" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:51Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T13:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T13:05:51Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c315880b179102 namespace: kserve-ci-e2e-test resourceVersion: "24541" uid: 8a6f90ae-96a7-44c2-b0db-4df344a30da7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T13:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:51Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c315880c9a5c67 namespace: kserve-ci-e2e-test resourceVersion: "24543" uid: d7899034-2aad-4962-9005-2ea45b86ea9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T13:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:51Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c31588100c7e87 namespace: kserve-ci-e2e-test resourceVersion: "24544" uid: c8b1a298-c611-418f-b11b-1e9022e1d1b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T13:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:51Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c315881086a975 namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: eeaedc06-fc32-4046-8726-5694f243e968 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c31589433da26f namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: f53fc197-68fa-48cb-9e9c-26dfcb17c977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c31589463a63db namespace: kserve-ci-e2e-test resourceVersion: "24587" uid: 0056d72d-6f43-498b-a7a1-1b47749f8851 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c3158946af1559 namespace: kserve-ci-e2e-test resourceVersion: "24588" uid: 4859c2a1-0be3-42a4-a0f1-f3ffaecaa5ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c3158946bad2a7 namespace: kserve-ci-e2e-test resourceVersion: "24590" uid: 2f650021-fa93-4c3e-88ad-9f52b274b9e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c3158949e3dbe4 namespace: kserve-ci-e2e-test resourceVersion: "24593" uid: ddd98b06-e448-48af-b012-c4ce56ec3289 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:05:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:05:56Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c315894a577d27 namespace: kserve-ci-e2e-test resourceVersion: "24594" uid: c9ab3b7b-2e88-44d9-9cc5-c9b4778f7d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:07:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:30Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c3159f1ec62913 namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: a213647d-2310-4d66-af12-def7c9bb8bc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1431c054-5578-49e9-8d3d-b99fc97bf144 kind: Event lastTimestamp: "2026-07-17T13:07:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:07:30Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh.18c3159f1ec6abcc namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: 24c10adb-0084-418f-a92d-1209fa4d1e42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "24503" uid: 3677ed47-9a61-4cd2-a13b-3fe22e350c5d kind: Event lastTimestamp: "2026-07-17T13:05:50Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-7lhsh' metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c31587cc9258bf namespace: kserve-ci-e2e-test resourceVersion: "24513" uid: 67938131-3ff8-4bff-970b-53b9d4bd41b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "24501" uid: 82b2e610-1c19-4dc7-bc60-e0743e0425c7 kind: Event lastTimestamp: "2026-07-17T13:05:50Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants-predictor.18c31587c821a304 namespace: kserve-ci-e2e-test resourceVersion: "24504" uid: e302dbfc-294b-43a3-b018-2453f1e5e26a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "24502" uid: 0173dc3d-34f0-4955-8632-118818d79296 kind: Event lastTimestamp: "2026-07-17T13:06:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:06:35Z" name: isvc-autogluon-v2-variants-predictor.18c3158b4702a305 namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: 554402a6-d361-44d4-9521-ec9e7811ac5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "24502" uid: 0173dc3d-34f0-4955-8632-118818d79296 kind: Event lastTimestamp: "2026-07-17T13:06:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T13:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:06:35Z" name: isvc-autogluon-v2-variants-predictor.18c3158b47796532 namespace: kserve-ci-e2e-test resourceVersion: "24861" uid: 96a8865e-0680-41dc-b754-1a081d06b519 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:06:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 0173dc3d-34f0-4955-8632-118818d79296 kind: Event lastTimestamp: "2026-07-17T13:07:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:07:20Z" name: isvc-autogluon-v2-variants-predictor.18c31595c46f3df7 namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 0a42a584-a255-4d7b-8ca9-1da40bed9759 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:06:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 0173dc3d-34f0-4955-8632-118818d79296 kind: Event lastTimestamp: "2026-07-17T13:07:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T13:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:07:20Z" name: isvc-autogluon-v2-variants-predictor.18c31595c4cfd365 namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: ca193d0f-c98d-4080-8623-e1c38aa10e1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "24494" uid: 044c2915-3e7b-4c46-9cf1-12b87cfe795c kind: Event lastTimestamp: "2026-07-17T13:07:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:07:26Z" name: isvc-autogluon-v2-variants.18c31587c8264456 namespace: kserve-ci-e2e-test resourceVersion: "25227" uid: 0c05164a-7bec-4fc4-b5a8-352a2f6881c0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "24494" uid: 044c2915-3e7b-4c46-9cf1-12b87cfe795c kind: Event lastTimestamp: "2026-07-17T13:05:50Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants.18c31587c8b4d46b namespace: kserve-ci-e2e-test resourceVersion: "24520" uid: 2f7d40d6-71b8-48df-b5c5-2b528556aecb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T13:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "24494" uid: 044c2915-3e7b-4c46-9cf1-12b87cfe795c kind: Event lastTimestamp: "2026-07-17T13:05:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:05:50Z" name: isvc-autogluon-v2-variants.18c31587c8b524a8 namespace: kserve-ci-e2e-test resourceVersion: "24523" uid: 4863f51c-5138-4791-ae42-d8e2a9826466 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:07:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: 044c2915-3e7b-4c46-9cf1-12b87cfe795c kind: Event lastTimestamp: "2026-07-17T13:07:26Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-17T13:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:07:26Z" name: isvc-autogluon-v2-variants.18c3159e16e85597 namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: 785d1269-40c4-4230-9b80-dd5f8fee67ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-17T13:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "23542" uid: ac38fe2c-9d73-416b-8db1-cda3bad8a46d kind: Event lastTimestamp: "2026-07-17T13:05:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T13:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:05:41Z" name: isvc-autogluon-v2.18c3156a99161981 namespace: kserve-ci-e2e-test resourceVersion: "24404" uid: 6e04b60d-1b6b-48f0-888c-be85bcb13b95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "23542" uid: ac38fe2c-9d73-416b-8db1-cda3bad8a46d kind: Event lastTimestamp: "2026-07-17T13:03:44Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:03:44Z" name: isvc-autogluon-v2.18c3156a99a65791 namespace: kserve-ci-e2e-test resourceVersion: "23557" uid: 2a647ed1-f154-4965-8884-e8a2e7019f87 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "23542" uid: ac38fe2c-9d73-416b-8db1-cda3bad8a46d kind: Event lastTimestamp: "2026-07-17T13:03:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T13:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:03:44Z" name: isvc-autogluon-v2.18c3156a99a6a85d namespace: kserve-ci-e2e-test resourceVersion: "23559" uid: 8fc0aa19-5f4b-41b1-b254-d49a2b358887 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "24401" uid: ac38fe2c-9d73-416b-8db1-cda3bad8a46d kind: Event lastTimestamp: "2026-07-17T13:05:41Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-17T13:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:05:41Z" name: isvc-autogluon-v2.18c31585a4e29ea7 namespace: kserve-ci-e2e-test resourceVersion: "24402" uid: 18cf6b1f-a68a-45ee-9142-97bfa9fbcd1c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T12:59:14.244558Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-84dd4b499f-8sqxq to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T12:59:14Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152b91e8a1f0 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 5a0ae426-a8c7-48f6-b220-65a5d97aea6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:14Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152b9a49b59e namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 5bea1ec1-a260-4b05-9c66-c6b4428e5bc7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:15Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T12:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T12:59:15Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152bcfa19b8e namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 8b0e7b8b-d363-4627-add1-a453b4bef605 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T12:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:15Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152bd133ad0c namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 65e4f444-4d8e-4325-94d6-d3702a1b9ad6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T12:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:15Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152bd4415b77 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 2dc071f5-6f98-469a-a99b-33a812f6e8cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T12:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:15Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152bd4bcd381 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 686ea8e1-27c9-45dd-87e8-1c6af559865f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1708" already present on machine metadata: creationTimestamp: "2026-07-17T12:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:18Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152ca5fa32aa namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: ff280e83-aef0-4a25-8cb1-87b7a98ab6e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T12:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:18Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cab960692 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: bf85d5a3-2cb0-4db8-90e9-ad55767e9d46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T12:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:18Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cac29ac37 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: ff4310d6-9f77-4c4c-894c-f957204a1965 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T12:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:18Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cac37630f namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: f5568e8f-60ce-49b5-b211-f4b42146c698 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:19Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cafd5ea7c namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 2f535311-f9f6-494d-ba8a-01ecb5f06391 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:19Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cb04c3a9e namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: b8697dc7-2064-4a3c-9520-bb19d6270df9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:19Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:6b71d408e980eff10059ec23e7854b4c9a37ce3f03bc65f1239585c87356b1c1" already present on machine metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:19Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cb0572b9f namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 903bb34d-e7ea-4ea5-8529-ebf034a29c26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:19Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:19Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cb39004a0 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: fa81327b-bd1b-4086-9d3f-77b907991eb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T12:59:19Z" message: Started container agent metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:19Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152cb407eb84 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 9f2b6d58-bbc9-4843-b6ae-fc8e986d7ecd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T12:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:00:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-17T12:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:56Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152ce20dc9da namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: 41866611-60b8-4363-afc1-5d75233cde2b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-17T12:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:00:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-17T12:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:56Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c3152d1de5f972 namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 838a9d2e-5fa3-4dbf-a6fb-6242ac9a8ccf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c31541bd546c06 namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: 313111c8-bfc2-4b44-a62e-3b8d5c21bd67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c31541bd54bae2 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: d2fa5980-37ad-47d5-8d10-52093c616083 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c31541bd54ddb1 namespace: kserve-ci-e2e-test resourceVersion: "22195" uid: 5f4e8b83-8b13-45be-a122-8a60f3bf09f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T13:00:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-84dd4b499f-8sqxq namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 4a1ff06b-005d-4538-bc3b-d73d6d729e44 kind: Event lastTimestamp: "2026-07-17T13:01:01Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-17T13:00:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:01:01Z" name: isvc-logger-predictor-84dd4b499f-8sqxq.18c315424e2c47eb namespace: kserve-ci-e2e-test resourceVersion: "22335" uid: cb7af83c-e5ab-4289-90bb-b86676f82cb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-84dd4b499f namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: a3be661c-2875-4634-944a-c24ebbe0d740 kind: Event lastTimestamp: "2026-07-17T12:59:14Z" message: 'Created pod: isvc-logger-predictor-84dd4b499f-8sqxq' metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:14Z" name: isvc-logger-predictor-84dd4b499f.18c3152b915a1a06 namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: 256f9458-5835-4936-bde5-d9d416380656 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 6dfbf6fd-e474-4bf4-84a5-5f40cf350313 kind: Event lastTimestamp: "2026-07-17T12:59:14Z" message: Scaled up replica set isvc-logger-predictor-84dd4b499f from 0 to 1 metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:14Z" name: isvc-logger-predictor.18c3152b8d6b161d namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: f995ce71-5f4e-4443-a1d8-03b4161c9b41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:59:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 373a9ad2-6c3e-400f-82cc-a554d5b916fa kind: Event lastTimestamp: "2026-07-17T12:59:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:44Z" name: isvc-logger-predictor.18c3152f0bf77e13 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 558bbf76-fea6-423e-981e-342af1b2b566 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:59:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 373a9ad2-6c3e-400f-82cc-a554d5b916fa kind: Event lastTimestamp: "2026-07-17T12:59:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:44Z" name: isvc-logger-predictor.18c3152f0c381470 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 923b5315-0d58-4b65-a84b-1f7160ab3fa0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T12:59:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 373a9ad2-6c3e-400f-82cc-a554d5b916fa kind: Event lastTimestamp: "2026-07-17T13:00:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:00:44Z" name: isvc-logger-predictor.18c3153609a0c9f0 namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 0afc79c9-ce30-4902-ba3b-420f76a0784f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T12:59:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 373a9ad2-6c3e-400f-82cc-a554d5b916fa kind: Event lastTimestamp: "2026-07-17T13:00:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T13:00:44Z" name: isvc-logger-predictor.18c315360a04d96e namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: 1522d156-5586-44e6-997d-2cbd4258852d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T12:59:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: dec86883-c211-40a6-8432-0fc2c1a4e53d kind: Event lastTimestamp: "2026-07-17T13:00:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:00:36Z" name: isvc-logger.18c3152b8d7cba3c namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 32158047-a61c-4c1a-815c-e7d0d3be92d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: dec86883-c211-40a6-8432-0fc2c1a4e53d kind: Event lastTimestamp: "2026-07-17T12:59:14Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T12:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:59:14Z" name: isvc-logger.18c3152b8e17823d namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 6167de99-31fe-4ac6-8014-caa8c314ebcd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: dec86883-c211-40a6-8432-0fc2c1a4e53d kind: Event lastTimestamp: "2026-07-17T13:00:36Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-17T13:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T13:00:36Z" name: isvc-logger.18c3153ea2ae01ab namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: aa1a7d05-d56a-4c02-b21e-1a28d7a1163b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T12:57:18.924222Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20146" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31510b847fedd namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: 2ee5334f-82c2-41c8-8800-2c69b91222ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:19Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T12:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T12:57:19Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31510d22c1024 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: 46581e35-8cbc-4701-9b8b-d2d36cc6f31a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T12:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:19Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31510d397667a namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: bad6dfe1-5498-42c9-a939-a6fab50a6248 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T12:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:19Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31510d6dc6e97 namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 649bd685-9bbf-431a-b1ca-b9c4e356fb57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T12:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:19Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31510d75549a5 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 8f9ccea3-59c9-4423-8a40-af8a9f9192c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1708" already present on machine metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c1216217 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: fcad7515-5d67-4665-990d-0f20137d5cc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c4da1fdb namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 4ab948f9-e352-4b9f-a4b3-9c5b7376fe82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c556e144 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 9cd49373-47ca-4eaa-9ee2-2bad736b479f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c5648494 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: d0de64d1-06a5-4ed7-8267-38887d02296c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c853d9fa namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: f33c973f-2eb8-46c7-aedd-540a1d7e6c9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c8cc0f29 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: fe5da248-b87e-48ce-af9e-3d1919a1dacd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:6b71d408e980eff10059ec23e7854b4c9a37ce3f03bc65f1239585c87356b1c1" already present on machine metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511c8d713e9 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: dafdd1ba-710b-4c0f-9e43-fae819c52ad0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511cbdb1e43 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: d559ecca-9ce9-4253-bde5-9820e9baa457 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:57:23Z" message: Started container agent metadata: creationTimestamp: "2026-07-17T12:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:23Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511cc547c5f namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 2aa9f5f5-204c-4404-8ccb-fd99d18dd07b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T12:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:59:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-17T12:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:01Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31511fd4e1e23 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 824c0776-9f0b-424e-9681-7333eceb1b9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-17T12:57:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:59:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-17T12:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:01Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c315123927e32f namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: d565d3cd-31d6-40b6-812c-2d8179473dd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:54Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31526e14560c9 namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: c324e798-b92f-48d2-b5cf-4d4355c33fcb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:54Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31526e1468415 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 85acadc0-ad09-47b5-a9e7-38fc27a16981 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:54Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31526e14690fa namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: e09a003b-33d0-4f3c-bb5f-649367e75a61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: e0e9ad9e-af60-4f29-88f1-96dda14d647e kind: Event lastTimestamp: "2026-07-17T12:59:11Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-17T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:59:11Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn.18c31527696f42b9 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: e64e7cfe-e572-4128-a30b-497d6420af51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-58f64b96c namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 40c81a1f-ec64-4780-8e34-497c2c5a08bc kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-58f64b96c-6mnfn' metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-custom-predictor-58f64b96c.18c31510b7b3450b namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 077b4476-85d2-4c7f-a780-5e429edaa185 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: acd6c5ee-096d-4a0b-a448-01fd5e300b84 kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-58f64b96c from 0 to 1 metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-custom-predictor.18c31510b314a807 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 089799f1-8ad1-48dd-8d8e-3391db4c01b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 39867deb-7648-4853-884c-6abcdf0edf18 kind: Event lastTimestamp: "2026-07-17T12:58:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:03Z" name: isvc-sklearn-batcher-custom-predictor.18c31514321466f4 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 3a8bc323-3fa4-44b1-b9ef-832ffdc2702c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 39867deb-7648-4853-884c-6abcdf0edf18 kind: Event lastTimestamp: "2026-07-17T12:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:03Z" name: isvc-sklearn-batcher-custom-predictor.18c31514328d26e5 namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 251b9c10-ac92-4cb0-b698-f4f3ea5465ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T12:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 39867deb-7648-4853-884c-6abcdf0edf18 kind: Event lastTimestamp: "2026-07-17T12:58:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:48Z" name: isvc-sklearn-batcher-custom-predictor.18c31517b1421c78 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 862ad24d-ea9a-4884-b009-a79ec90d9bd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-17T12:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 39867deb-7648-4853-884c-6abcdf0edf18 kind: Event lastTimestamp: "2026-07-17T12:58:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:48Z" name: isvc-sklearn-batcher-custom-predictor.18c31517b1a3c35b namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4fba8724-9799-4c61-8bb8-46b39216f95c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 1311c319-36cd-4e9f-9bd9-77c4b1198199 kind: Event lastTimestamp: "2026-07-17T12:58:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:58:41Z" name: isvc-sklearn-batcher-custom.18c31510b315d9d0 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 98efc4e4-4dd6-490a-aa2e-32cd3d74c051 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 1311c319-36cd-4e9f-9bd9-77c4b1198199 kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-custom.18c31510b3919581 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: b2f2b527-afa9-4c0d-a454-daaa5ef9225b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 1311c319-36cd-4e9f-9bd9-77c4b1198199 kind: Event lastTimestamp: "2026-07-17T12:58:41Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-17T12:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:58:41Z" name: isvc-sklearn-batcher-custom.18c31523f4d36b5b namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: ddcb0a15-1529-4b64-bc44-cbd008d3907e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T12:55:33.814622Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T12:55:33Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f83f42c2e7 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: e3375595-f232-471e-9201-56553c079b53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:34Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T12:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T12:55:34Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f85877225f namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: a415dd9c-39d5-42a3-95c7-f60c8270cf40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:97ab9417b2aa429ec98ee8db7bc547e651745755de331aed4b07eef73ebde9e8" already present on machine metadata: creationTimestamp: "2026-07-17T12:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:34Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f85a5462a5 namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: ac72d92c-fac5-4ce5-86e4-c1494a295e58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-17T12:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:34Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f85cda815a namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: e8bc8455-9803-425c-a6d9-84959dc9506b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-17T12:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:34Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f85d4b540d namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: af2c0e12-272b-4dbb-9f0f-12ff1f80872f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1708" metadata: creationTimestamp: "2026-07-17T12:55:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:37Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314f93297d221 namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: 5bfef398-f5c7-4ac7-b5da-bc2579d6dadb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:51Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1708" in 13.959s (13.959s including waiting). Image size: 1555968935 bytes.' metadata: creationTimestamp: "2026-07-17T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:51Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fc72a2e903 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 3527bcc4-6e76-4cb3-b4ce-62a875229dc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:51Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fc7677e7b8 namespace: kserve-ci-e2e-test resourceVersion: "19464" uid: 21b61489-45e5-4da0-a839-8d12273381c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:51Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fc76e9b20d namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 6ce846b3-0e15-4da3-9825-0e51c22f160e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-17T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:51Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fc76f73ea4 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: d50b014f-5ce6-4c0c-a094-1243dea4abc7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.666s (2.666s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-17T12:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:54Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fd15ed4976 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 8a70f3f0-9507-4b07-b780-c06d7b76f112 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T12:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:54Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fd18c9e4c8 namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 48b6b00e-93ad-4fbc-a021-8bb4f1afdfc4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:54Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fd19447243 namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: 50d1dc2d-af49-4975-9734-09e2331d5e5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:54Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:6b71d408e980eff10059ec23e7854b4c9a37ce3f03bc65f1239585c87356b1c1" metadata: creationTimestamp: "2026-07-17T12:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:54Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fd1954abb6 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: ee69f510-e3c3-45cf-a004-700040108fed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:6b71d408e980eff10059ec23e7854b4c9a37ce3f03bc65f1239585c87356b1c1" in 2.263s (2.263s including waiting). Image size: 240005729 bytes.' metadata: creationTimestamp: "2026-07-17T12:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:56Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fda0426285 namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: b98fc67a-8f95-48d8-9759-f1f739707cfb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:56Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-17T12:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:56Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fda32f4414 namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: 37662da8-d58b-4cd1-8fec-c407d56ca3e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:55:56Z" message: Started container agent metadata: creationTimestamp: "2026-07-17T12:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:55:56Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fda3a5bd93 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: ede14606-6031-439c-9c0d-a36ecbe2c4ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-17T12:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-17T12:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:30Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fda5911e80 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 607c8647-7c46-4a6e-9c8f-44a52976fa10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-17T12:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-17T12:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:20Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c314fde16352c2 namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 46820a7d-87bd-4589-af9e-bfddec92402c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c31510b19b3b88 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: b3279218-50a3-4af8-ad44-7b60ea1fb8fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c31510b19b88fb namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: d6281529-71b0-4df9-a959-aba96814579f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:18Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c31510b19be3a3 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: f400b88c-6325-436c-a702-ef14970e65dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T12:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 26c4e402-5a31-4b5f-9863-a71855ea1c2b kind: Event lastTimestamp: "2026-07-17T12:57:29Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-17T12:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:57:29Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9.18c31510bd957a93 namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 5c3175db-2a94-479a-9965-6b3ae68947f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-7f9d5b8f96 namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: a3c9561b-4e1e-413c-b697-b197ca34c98f kind: Event lastTimestamp: "2026-07-17T12:55:33Z" message: 'Created pod: isvc-sklearn-batcher-predictor-7f9d5b8f96-cdkl9' metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:55:33Z" name: isvc-sklearn-batcher-predictor-7f9d5b8f96.18c314f83e421011 namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: eeca1a0d-8f66-4f93-8c45-8413f5977118 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: fc4a159c-0e77-4595-8b0b-931ad52b2ee8 kind: Event lastTimestamp: "2026-07-17T12:55:33Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-7f9d5b8f96 from 0 to 1 metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:55:33Z" name: isvc-sklearn-batcher-predictor.18c314f836b211dd namespace: kserve-ci-e2e-test resourceVersion: "19294" uid: f957a61e-42b5-4529-9b8d-cfbd97abe8ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T12:55:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "19300" uid: 540b6c2a-00f3-41bb-85bc-286654b8206e kind: Event lastTimestamp: "2026-07-17T12:56:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:56:33Z" name: isvc-sklearn-batcher-predictor.18c314fbbc3ee8a5 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 3f5ff91c-4b54-41cd-ba75-1dc866791541 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T12:55:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "19300" uid: 540b6c2a-00f3-41bb-85bc-286654b8206e kind: Event lastTimestamp: "2026-07-17T12:56:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:56:33Z" name: isvc-sklearn-batcher-predictor.18c314fbbcaed7e9 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 39e39fab-18f8-44c5-9aeb-71885ae6956c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T12:56:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 540b6c2a-00f3-41bb-85bc-286654b8206e kind: Event lastTimestamp: "2026-07-17T12:57:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:57:03Z" name: isvc-sklearn-batcher-predictor.18c31502baa06e76 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 36602577-0811-4da1-8a90-1dd42b3fdced reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-17T12:56:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 540b6c2a-00f3-41bb-85bc-286654b8206e kind: Event lastTimestamp: "2026-07-17T12:57:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:57:03Z" name: isvc-sklearn-batcher-predictor.18c31502bb086dd7 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 2cffb6ae-d99e-491c-9a1e-c3a0d92326c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-17T12:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 12d53191-0024-46d7-8275-6f2c8bfd0f54 kind: Event lastTimestamp: "2026-07-17T12:57:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:57:10Z" name: isvc-sklearn-batcher.18c314f83d4d0bf4 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 6357f90f-d87c-4c7c-b1c6-46a67f69c1b1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 12d53191-0024-46d7-8275-6f2c8bfd0f54 kind: Event lastTimestamp: "2026-07-17T12:55:33Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:55:33Z" name: isvc-sklearn-batcher.18c314f83dc74f7c namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 77701f7d-688b-4c81-b8a4-8e21abc3a072 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 12d53191-0024-46d7-8275-6f2c8bfd0f54 kind: Event lastTimestamp: "2026-07-17T12:55:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T12:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:55:33Z" name: isvc-sklearn-batcher.18c314f83dc79c88 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 3941e1df-0055-4b62-9685-1f72b8f8c905 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:57:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 12d53191-0024-46d7-8275-6f2c8bfd0f54 kind: Event lastTimestamp: "2026-07-17T12:57:10Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-17T12:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:57:10Z" name: isvc-sklearn-batcher.18c3150eaba58fff namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 37f67a9b-8fc1-4baa-8df9-975aabcdd017 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-17T12:58:54.182481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-2c4kj to ip-10-0-137-99.ec2.internal metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c31526e61da4be namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: cd7e6595-9282-4c0b-bd06-d74a8b46b5d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7869fb68d9-j7fh7 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c31526ffc889fe namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: a0ab8d1b-f0b1-4f05-9091-c4c31e57a9e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c31527013690dd namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: e8998174-a9cd-4123-84cf-3ba582d3121e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.161s (1.161s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c315274675a6da namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: b5f08edd-7110-4642-b4dc-0f39951fbe0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c3152749fd598d namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 31f796c7-fa6c-425b-baf7-4a2264a257e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c315274a709038 namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 10562813-964b-41ec-a525-c3cc04533b6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c315274a7b31e2 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 9a7b1058-104c-4ddc-aa9e-e2fd33fcb589 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c315274d9c9e26 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: a70e09aa-0825-4db5-9c84-f9f1fe583534 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T12:58:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T12:58:55Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c315274e155b70 namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: e30bf646-e7d6-479d-95cb-6ac3f377baf4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c31541c2dac726 namespace: kserve-ci-e2e-test resourceVersion: "22222" uid: ebcba44e-70fb-4ad3-99f8-9d27dd3ab0e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T13:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-2c4kj namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 752cd404-5062-457f-9223-152507394688 kind: Event lastTimestamp: "2026-07-17T13:00:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-17T13:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-17T13:00:49Z" name: message-dumper-predictor-6575bcfdfd-2c4kj.18c31541c2dc06ac namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: cbe5fb06-b756-408c-be9e-21da3d0c4a56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-99.ec2.internal source: component: kubelet host: ip-10-0-137-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 475ea769-159e-46e9-857e-6d5f18a5a52e kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-2c4kj' metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper-predictor-6575bcfdfd.18c31526e56b2a0d namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: f95d93b0-4879-413e-b32e-850fb11fd116 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 1ec8d7e4-188f-4598-b9e6-df4164687c31 kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper-predictor.18c31526e1345670 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 7f53e1ef-f93c-4a9a-bed0-c8cabf4699f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:59:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 24918f30-1667-47b1-8bd2-6637cbc813fe kind: Event lastTimestamp: "2026-07-17T12:59:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:39Z" name: message-dumper-predictor.18c3152a6012cbf9 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: a02ff78e-82d7-4e46-b844-64a656333473 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-17T12:59:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 24918f30-1667-47b1-8bd2-6637cbc813fe kind: Event lastTimestamp: "2026-07-17T12:59:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-17T12:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:39Z" name: message-dumper-predictor.18c3152a60c0e9c2 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: a280dd56-c080-46ba-b035-162ed17d8895 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 24918f30-1667-47b1-8bd2-6637cbc813fe kind: Event lastTimestamp: "2026-07-17T12:59:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:24Z" name: message-dumper-predictor.18c3152de0ab3978 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: a75b39e2-b7f3-4cf5-bdb6-b8def140d69e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 24918f30-1667-47b1-8bd2-6637cbc813fe kind: Event lastTimestamp: "2026-07-17T12:59:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-17T12:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-17T12:59:24Z" name: message-dumper-predictor.18c3152de1ae5211 namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: 1d32fe06-7582-4511-80c8-0b351557dc0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 3ef447b0-749a-4228-909e-30c5dbdc5685 kind: Event lastTimestamp: "2026-07-17T12:59:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:59:04Z" name: message-dumper.18c31526e16ad6d1 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: a3f107c0-dad2-40e3-8993-879f3c9cf04f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:58:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 3ef447b0-749a-4228-909e-30c5dbdc5685 kind: Event lastTimestamp: "2026-07-17T12:58:54Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-17T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:58:54Z" name: message-dumper.18c31526e1f8a893 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 131d06dd-9893-400a-bb1b-913d355df575 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-17T12:59:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 3ef447b0-749a-4228-909e-30c5dbdc5685 kind: Event lastTimestamp: "2026-07-17T12:59:04Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-17T12:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-17T12:59:04Z" name: message-dumper.18c3152964462406 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: c5128be0-eace-4894-9cc8-21e64bc6ea5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "26696"