--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:24:18.386047Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22676" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck to ip-10-0-130-13.ec2.internal metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b487c158352 namespace: kserve-ci-e2e-test resourceVersion: "22682" uid: 6f142754-9871-4ca1-872b-c0075c119a1e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22687" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4896a0b30a namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: 4704a99f-6dde-49dc-99da-95ec7ca7938a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4898663d9f namespace: kserve-ci-e2e-test resourceVersion: "22695" uid: e5de547c-92e9-41ec-ac3a-747cbb7a9908 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b489b8effae namespace: kserve-ci-e2e-test resourceVersion: "22697" uid: e918c2bc-b100-4884-96fa-943f18cfc6ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b489c13cc93 namespace: kserve-ci-e2e-test resourceVersion: "22698" uid: 8c876063-62bd-4c43-99d7-6a4ef5bad9d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b079c52ef namespace: kserve-ci-e2e-test resourceVersion: "22771" uid: 6660052c-9185-4971-be59-540e7cf9c1c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b0aa4cbf8 namespace: kserve-ci-e2e-test resourceVersion: "22773" uid: 6559184e-09a8-4fb2-8fef-d13c320ee662 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b0b27f111 namespace: kserve-ci-e2e-test resourceVersion: "22774" uid: 83815377-9e8b-4a7f-8988-3888d598612b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b0b33259c namespace: kserve-ci-e2e-test resourceVersion: "22775" uid: 3e627ca6-33df-4731-a87a-971439e277a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b0e9f6e55 namespace: kserve-ci-e2e-test resourceVersion: "22778" uid: 6f6a09de-6dd9-4c53-8f45-1c46ce3c1376 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b0f4a666c namespace: kserve-ci-e2e-test resourceVersion: "22780" uid: ffb3cb10-7bce-4b8f-96eb-e1e83defd06b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:37Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4b7f831f6d namespace: kserve-ci-e2e-test resourceVersion: "22846" uid: 66f37a80-c091-4d63-9581-e0b6e16be86b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:24:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:48Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4f7b75d107 namespace: kserve-ci-e2e-test resourceVersion: "22946" uid: 9434f382-64d1-4474-83af-33bf02c92f0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 1451e58e-aeb9-457b-901f-f6be1d1d11e2 kind: Event lastTimestamp: "2026-07-07T15:24:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:24:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:48Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck.18c00b4f7b766834 namespace: kserve-ci-e2e-test resourceVersion: "22947" uid: c6b12144-a8db-4a34-8183-a286fc1c108a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: c85393db-ef92-44ad-a487-a2c724649a69 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-xknck' metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c00b487a0b96b9 namespace: kserve-ci-e2e-test resourceVersion: "22677" uid: ce8bb7da-bade-486c-bb12-6a54fb7d18a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 24149ff1-bcfd-4b25-834c-eba1b3fc6db1 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1-predictor.18c00b4873ef7bdf namespace: kserve-ci-e2e-test resourceVersion: "22666" uid: e87c2a3a-0d9b-45f8-87db-44c8b4a4d874 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:24:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22664" uid: 66ba2158-1e08-4edd-93d8-26cf5a7ac9ee kind: Event lastTimestamp: "2026-07-07T15:24:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:24:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:48Z" name: isvc-autogluon-ts-v1-predictor.18c00b4bf2fa0165 namespace: kserve-ci-e2e-test resourceVersion: "22931" uid: 5117415b-fd70-4cc3-8371-fa813d828e64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:24:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22664" uid: 66ba2158-1e08-4edd-93d8-26cf5a7ac9ee kind: Event lastTimestamp: "2026-07-07T15:24:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:24:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:48Z" name: isvc-autogluon-ts-v1-predictor.18c00b4bf3756eae namespace: kserve-ci-e2e-test resourceVersion: "22932" uid: 0cafc8ae-ad82-4d62-be17-2d077eac6110 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22656" uid: b1ccb518-6961-455c-bfe5-dbbf840577f7 kind: Event lastTimestamp: "2026-07-07T15:24:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:47Z" name: isvc-autogluon-ts-v1.18c00b4873f9e50a namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: 1f82affe-a127-409f-b59d-5b294afa52df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22656" uid: b1ccb518-6961-455c-bfe5-dbbf840577f7 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1.18c00b4874866772 namespace: kserve-ci-e2e-test resourceVersion: "22685" uid: 4d16e352-447a-44cb-95ab-8b50279d69ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:24:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22656" uid: b1ccb518-6961-455c-bfe5-dbbf840577f7 kind: Event lastTimestamp: "2026-07-07T15:24:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:18Z" name: isvc-autogluon-ts-v1.18c00b487486aa98 namespace: kserve-ci-e2e-test resourceVersion: "22686" uid: 9da4d107-85b4-455d-aa01-5766d248711c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: b1ccb518-6961-455c-bfe5-dbbf840577f7 kind: Event lastTimestamp: "2026-07-07T15:24:47Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-07T15:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:47Z" name: isvc-autogluon-ts-v1.18c00b4f3f43d538 namespace: kserve-ci-e2e-test resourceVersion: "22919" uid: 70dd028f-891d-4c1a-8021-7a8dcb8a9cf9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:11:46.827184Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k to ip-10-0-130-13.ec2.internal metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:11:46Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00a997faf6cb5 namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: ac46c827-28a4-418f-85b5-48d8df0452b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:11:47Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:11:47Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00a99991b8c77 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 055d6759-2a5e-4c37-bda3-2712a073a706 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:11:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:47Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00a999ad2bf22 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: 9eb23424-6bae-4c0b-b593-da00b43807aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:11:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:47Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00a999dcfe3fd namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: 3f3c59e7-4244-4d8a-9b7e-6b93392c475d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:11:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:47Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00a999e4073a5 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: 1ac993de-b31a-44ce-a09e-02b8a90643cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:17Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00aa09476253c namespace: kserve-ci-e2e-test resourceVersion: "17368" uid: fdb21f92-db85-4ed4-8d9a-f882d7d9c252 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:44Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m27.656s (2m27.656s including waiting). Image size: 8939170587 bytes.' metadata: creationTimestamp: "2026-07-07T15:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:44Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac2f57a7bf3 namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: 48ab6eeb-f4a3-43a7-b64a-e71c836b61fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:45Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac2fbb02d8b namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: 2dbf137b-6b37-448e-ba4a-c2985640d168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:45Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac2fcbf8388 namespace: kserve-ci-e2e-test resourceVersion: "18394" uid: 883f69e8-45c4-49c2-a69a-480e5d37574b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:45Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac2fccccd3a namespace: kserve-ci-e2e-test resourceVersion: "18395" uid: 754041be-460b-4dd9-a017-8917df4a2b46 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.291s (3.291s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac3c103934a namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: 1a32a554-9075-40fd-a9c6-79c7c0b9b48a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac3c71d0a9d namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: 3c6b12fc-7066-4ab5-9ffc-7f8e11072d75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac3c813f4ca namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 53099ff5-e113-4fe8-b2ef-ed8b71b58929 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:14:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:55Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac421695c81 namespace: kserve-ci-e2e-test resourceVersion: "18478" uid: cd7f70c7-dd42-44cd-8d90-a4900e4f31f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:15:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac818188163 namespace: kserve-ci-e2e-test resourceVersion: "18580" uid: c2fd501f-1ce4-4282-8a50-346fd34175fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 5f60202a-6a6c-45e2-82e3-ade8129035c8 kind: Event lastTimestamp: "2026-07-07T15:15:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k.18c00ac818190601 namespace: kserve-ci-e2e-test resourceVersion: "18581" uid: 0c4beea8-8b49-4f93-a3bc-89f90eea0dda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: cbf5e1ee-22d3-410a-98db-9c4558310eb6 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-whq2k' metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:46Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c00a997ea96a1e namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 709f0d48-91eb-45e7-8c73-503dcbc51b1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: aca1a558-18c4-4e5c-bfa3-0fb3cb794ed4 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:46Z" name: isvc-autogluon-v1-predictor.18c00a9979379ec0 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 77fd09d9-ab62-4f1e-a6e1-c44d1fc1dd2e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-07T15:12:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 1ecae883-91ff-40f8-b25d-51d900e0202c kind: Event lastTimestamp: "2026-07-07T15:15:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:15:01Z" name: isvc-autogluon-v1-predictor.18c00a9cf9009860 namespace: kserve-ci-e2e-test resourceVersion: "18519" uid: a93dd59b-f6d7-4f2b-bf45-3350d529cbfe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:12:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 1ecae883-91ff-40f8-b25d-51d900e0202c kind: Event lastTimestamp: "2026-07-07T15:14:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:46Z" name: isvc-autogluon-v1-predictor.18c00a9cf9795194 namespace: kserve-ci-e2e-test resourceVersion: "18411" uid: ec4f7622-d6e1-46f4-a86f-c5b711043d4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 6e9023a0-afc3-41f0-9a4e-79b182470333 kind: Event lastTimestamp: "2026-07-07T15:15:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:15:06Z" name: isvc-autogluon-v1.18c00a99793f664e namespace: kserve-ci-e2e-test resourceVersion: "18559" uid: ad217fc0-19cb-44f6-8d8a-a5be47e13ddf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 6e9023a0-afc3-41f0-9a4e-79b182470333 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:46Z" name: isvc-autogluon-v1.18c00a9979e781c8 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 0878fecf-4aa0-4642-91e6-0393023cd427 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 6e9023a0-afc3-41f0-9a4e-79b182470333 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:46Z" name: isvc-autogluon-v1.18c00a9979e7cf94 namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: 46e7d780-0ff6-487c-8047-559374d0be7b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18557" uid: 6e9023a0-afc3-41f0-9a4e-79b182470333 kind: Event lastTimestamp: "2026-07-07T15:15:06Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-07T15:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:15:06Z" name: isvc-autogluon-v1.18c00ac7e16df46f namespace: kserve-ci-e2e-test resourceVersion: "18558" uid: 5149d1d6-2858-4079-af27-4201ca15b590 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:22:13.014132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m to ip-10-0-130-13.ec2.internal metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b2b4b56517b namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 716ff635-b197-42f6-9d5e-a3c88d0fc3aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b2b64f1dd79 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8ece5cc5-31b4-4903-9b60-3c055c192df7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b2b66e2adf2 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 0d61d238-22fe-4243-b818-beeebfa52c16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b2b6a131eda namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 1471b694-148a-4e58-b4a9-6cd88eeacdce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b2b6a7a314d namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: b05af198-5750-4cbf-9860-eb6df5a851a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b302267ca56 namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: 8392e7c6-7f25-4c6b-93ed-236a417f4bd0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b30258e5ce2 namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: 96b907aa-fbcc-4ea8-aa4e-64e2141c5dde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b30260c1c30 namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: e2fb926d-4bb3-4a75-bfad-b3ad4fa4cca3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b302616d4e6 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 7c4751c0-b82a-4695-a45f-e45db83d4f39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b30290dc187 namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 4290d9f7-2488-426c-98f0-ca2c3493cb4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:22:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b30298b2cd0 namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: bff9de96-f46c-481e-9466-626ec06335ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:24:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b47462e8ef6 namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: a6e0d890-93ef-4938-afd0-49eb7fc20501 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 02c73ec1-9dab-4e5a-b93c-c30715334ba9 kind: Event lastTimestamp: "2026-07-07T15:24:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:24:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m.18c00b47462f2b43 namespace: kserve-ci-e2e-test resourceVersion: "22610" uid: 2505db97-7edd-473b-88d0-e910ffd22abf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 635ad1ff-d382-44fc-a8c8-ddc67b764f6c kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-cct6m' metadata: creationTimestamp: "2026-07-07T15:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c00b2b4ac7c492 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 98be2609-11fc-4100-8546-e3f976f924eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 9d0a47f1-fcb8-4498-af75-322e6a7ffab8 kind: Event lastTimestamp: "2026-07-07T15:22:12Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:12Z" name: isvc-autogluon-v2-noslash-predictor.18c00b2b45ec4cbb namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 5e8dcd02-f89d-4443-a508-96ca96328b56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:22:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6925d0ae-ad07-4068-947c-f6b5a4b358f0 kind: Event lastTimestamp: "2026-07-07T15:22:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:57Z" name: isvc-autogluon-v2-noslash-predictor.18c00b2ec4f9e8d2 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 75e2a723-dd4a-4b28-86e1-eaad65e09548 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:22:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6925d0ae-ad07-4068-947c-f6b5a4b358f0 kind: Event lastTimestamp: "2026-07-07T15:22:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:57Z" name: isvc-autogluon-v2-noslash-predictor.18c00b2ec5675a1e namespace: kserve-ci-e2e-test resourceVersion: "22066" uid: 210e0976-a5c0-4fe6-b368-50e0990cac7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:23:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 6925d0ae-ad07-4068-947c-f6b5a4b358f0 kind: Event lastTimestamp: "2026-07-07T15:24:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:13Z" name: isvc-autogluon-v2-noslash-predictor.18c00b39426becbc namespace: kserve-ci-e2e-test resourceVersion: "22594" uid: c7d52a81-b8e1-478f-a267-ecd4f0046a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:23:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 6925d0ae-ad07-4068-947c-f6b5a4b358f0 kind: Event lastTimestamp: "2026-07-07T15:24:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:24:13Z" name: isvc-autogluon-v2-noslash-predictor.18c00b3942e4f70c namespace: kserve-ci-e2e-test resourceVersion: "22595" uid: 2e716d40-ec94-478c-b18e-27b9ca31b5b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:22:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 8b7b300f-e230-4a07-be39-8cd949dd652e kind: Event lastTimestamp: "2026-07-07T15:24:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:05Z" name: isvc-autogluon-v2-noslash.18c00b2b45f671cd namespace: kserve-ci-e2e-test resourceVersion: "22538" uid: 53434955-41e6-4724-8ae7-3d508f4412cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:22:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 8b7b300f-e230-4a07-be39-8cd949dd652e kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash.18c00b2b467f086a namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 9661e020-aad1-4a86-92d3-2472c2533ddb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:22:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 8b7b300f-e230-4a07-be39-8cd949dd652e kind: Event lastTimestamp: "2026-07-07T15:22:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:13Z" name: isvc-autogluon-v2-noslash.18c00b2b467f6793 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 88f7d123-a918-4140-bfb8-a00172d7376d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:24:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22535" uid: 8b7b300f-e230-4a07-be39-8cd949dd652e kind: Event lastTimestamp: "2026-07-07T15:24:05Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-07T15:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:24:05Z" name: isvc-autogluon-v2-noslash.18c00b4576c04100 namespace: kserve-ci-e2e-test resourceVersion: "22536" uid: 06dfb466-6ce0-445d-ab90-de8014368299 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:15:12.266951Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18644" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-8p5bm to ip-10-0-140-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac954d987c9 namespace: kserve-ci-e2e-test resourceVersion: "18648" uid: 3dac7969-d0cd-4e73-acbe-94753026a7be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac96202e4e3 namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: 4c529085-8c59-434a-8e34-f29a05b81f9c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18655" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:13Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:15:13Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac99206292b namespace: kserve-ci-e2e-test resourceVersion: "18666" uid: f3cd9817-baf4-4099-bbf4-bf9b6f7e5170 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:13Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac9938d9af3 namespace: kserve-ci-e2e-test resourceVersion: "18668" uid: 32f076be-0186-41e9-b063-4e63cceead91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:13Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac9964ef9da namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: b426acdf-e053-4c74-975c-b16eab8ee5e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:13Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ac996c396c3 namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: ed244586-66c0-4341-b025-9bfafda424d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:15:43Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-07T15:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:15:43Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00ad08765c182 namespace: kserve-ci-e2e-test resourceVersion: "18883" uid: 075c2c7c-5416-4d94-9a85-c3137c3a262c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m26.46s (2m26.46s including waiting). Image size: 8939170587 bytes.' metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2a123b2c9 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cda6e305-a357-4427-b991-529db739eb06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2a7f04340 namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: 59b2c540-0252-4232-ae21-a2cfb915e156 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2a8f7931c namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 4b4ac187-52de-4fd0-a424-6d6c3dfdf270 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2a901bf85 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 2e103eb1-e891-4da6-9584-ee6f23607a84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2adbbab4c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: ce3682c5-fdb2-47ae-bb1a-9323c9e489e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:18:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:09Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00af2af10fbbf namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 0a1f4a52-e306-4e09-b4d4-84146494c551 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:19:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:52Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00b0a9ac93727 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 547185cb-f735-465f-a4f7-a651f70c7b36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:19:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:52Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00b0a9ac9ccf3 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 67f2d16e-85ad-46ad-b339-e48868d3f35b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-8p5bm namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: d731d089-94f4-4860-a873-f9bcd2c97169 kind: Event lastTimestamp: "2026-07-07T15:19:52Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:52Z" name: isvc-autogluon-v2-predictor-67b849d456-8p5bm.18c00b0aa4ed8c0a namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 42affb8a-8375-4444-9179-cb86a0eab8f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "18635" uid: 52fb4a08-e77b-4023-bbf8-286782d3681a kind: Event lastTimestamp: "2026-07-07T15:15:12Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-8p5bm' metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2-predictor-67b849d456.18c00ac95466c6d8 namespace: kserve-ci-e2e-test resourceVersion: "18645" uid: 377a9672-ab02-4f12-8065-641d3f8e5396 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: c3317248-4530-4b73-8fe4-d8997d8cb215 kind: Event lastTimestamp: "2026-07-07T15:15:12Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2-predictor.18c00ac94ff3d360 namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: da488622-1bae-4ae4-ade4-16b891ef85db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-07T15:15:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18634" uid: 26b8c94a-4e4d-4f2c-8c58-a00569d2a3c5 kind: Event lastTimestamp: "2026-07-07T15:18:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:15:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:27Z" name: isvc-autogluon-v2-predictor.18c00acccf0ce988 namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: 21213745-6760-40d2-b571-34334a6b6e95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:15:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18634" uid: 26b8c94a-4e4d-4f2c-8c58-a00569d2a3c5 kind: Event lastTimestamp: "2026-07-07T15:18:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:15:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:12Z" name: isvc-autogluon-v2-predictor.18c00acccf7e5d87 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 9a742c1f-9900-4b14-9cbc-443340855846 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:19:57.777892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm to ip-10-0-130-13.ec2.internal metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:19:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0bcea0acf6 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: b0497963-28a4-49e0-bd5a-ba9eb87b35d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:19:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-variants-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0bdc2a106d namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 25959cb1-291d-4379-b67b-3162d0879990 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:19:58Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:19:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0c0bc3c19d namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 30aea950-207b-49d7-8d87-9631aff1a1b6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:19:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0c0dc7eef9 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 3e678787-d67d-4cb7-b23e-c510e5eb8fa1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:19:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0c106dcc10 namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: cafcfe8a-776a-4b6c-8aad-fafe9732daf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:19:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b0c10e34719 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: bab0ed24-97c9-400a-8101-7b49f8faba50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b132465e5ab namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: eb973865-bda6-46c5-b7b8-a2979ff612c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b1327762b5c namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 47a697bb-4d7b-4358-9ac7-3f2298f67a50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b132839aac8 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: aa11bffc-3346-4ac4-bccc-f211135ad9a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b132843fd0c namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 36e9e35b-23a8-4b68-b09c-660717048cce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b132b230a8f namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: c0e3c914-4cdf-4401-ae33-5a49c74acec9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:20:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:29Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b132b86eebf namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 0ce8634b-5910-4c1b-b138-21939bc98b80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:22:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:07Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b2a1783fe26 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: ea3e1feb-d6de-4576-bee7-1f191a060402 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a745134e-0660-4dcb-87ae-dfc67c22e996 kind: Event lastTimestamp: "2026-07-07T15:22:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:07Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm.18c00b2a178495ec namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: c1d8a64e-8755-4283-ac3e-b06a87921f8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-13.ec2.internal source: component: kubelet host: ip-10-0-130-13.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 76af5aa9-9509-4259-8fe1-55047a6b0f77 kind: Event lastTimestamp: "2026-07-07T15:19:57Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-lh9bm' metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c00b0bcd9e7608 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 6215a700-929d-4895-8a21-eae2fec83bc7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 137f785b-1d41-4e54-a526-c3ab5bb8da1d kind: Event lastTimestamp: "2026-07-07T15:19:57Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:57Z" name: isvc-autogluon-v2-variants-predictor.18c00b0bc6d0ea56 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 83394b73-c360-4fd0-a2e3-8b6760663f87 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:20:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 8f18f49a-ad52-49e8-8656-0d3e69933941 kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:57Z" name: isvc-autogluon-v2-variants-predictor.18c00b0f46989794 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 1e735667-1389-40cb-82e8-53619b858a93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:20:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 8f18f49a-ad52-49e8-8656-0d3e69933941 kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:57Z" name: isvc-autogluon-v2-variants-predictor.18c00b0f470fa8fc namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 53933f5c-69ea-4834-ba38-70fd1b68d28b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:20:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 8f18f49a-ad52-49e8-8656-0d3e69933941 kind: Event lastTimestamp: "2026-07-07T15:21:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:21:57Z" name: isvc-autogluon-v2-variants-predictor.18c00b1645771506 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 419cb2a4-e37d-42d8-b0b6-cdb163fc15b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:20:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 8f18f49a-ad52-49e8-8656-0d3e69933941 kind: Event lastTimestamp: "2026-07-07T15:21:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:21:57Z" name: isvc-autogluon-v2-variants-predictor.18c00b1646184a98 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 5e5be217-1179-4dcf-b8dc-17fa157b8716 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:19:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 1c1112e8-17b5-4ebd-b730-d6bff9492db6 kind: Event lastTimestamp: "2026-07-07T15:22:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:02Z" name: isvc-autogluon-v2-variants.18c00b0bc6e41c99 namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: fd4db1ab-6d72-4878-a160-e115bd33696a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 1c1112e8-17b5-4ebd-b730-d6bff9492db6 kind: Event lastTimestamp: "2026-07-07T15:19:57Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:57Z" name: isvc-autogluon-v2-variants.18c00b0bc762ae48 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: e785433f-9d79-4925-b6d5-68c1771d8ba4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 1c1112e8-17b5-4ebd-b730-d6bff9492db6 kind: Event lastTimestamp: "2026-07-07T15:19:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:57Z" name: isvc-autogluon-v2-variants.18c00b0bc762fb85 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 535b6aa3-99a1-4af4-94b2-b5b0fc5ce354 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 1c1112e8-17b5-4ebd-b730-d6bff9492db6 kind: Event lastTimestamp: "2026-07-07T15:22:02Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-07T15:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:02Z" name: isvc-autogluon-v2-variants.18c00b28d310786e namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: fcec1cc9-9336-4ab7-a2c3-5665dd75438c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: 325b5e41-49de-405e-b785-7f72cdc3fb5f kind: Event lastTimestamp: "2026-07-07T15:19:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:44Z" name: isvc-autogluon-v2.18c00ac94ff22ad8 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 3d6e567f-80a9-4234-8e54-618ebcf3168b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: 325b5e41-49de-405e-b785-7f72cdc3fb5f kind: Event lastTimestamp: "2026-07-07T15:15:12Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2.18c00ac9507f0471 namespace: kserve-ci-e2e-test resourceVersion: "18652" uid: ecefe378-8ea9-487d-8fbe-fac3407781dd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:15:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: 325b5e41-49de-405e-b785-7f72cdc3fb5f kind: Event lastTimestamp: "2026-07-07T15:15:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:15:12Z" name: isvc-autogluon-v2.18c00ac9507f4f33 namespace: kserve-ci-e2e-test resourceVersion: "18654" uid: 44d7cc12-3bf8-46d6-896e-66efcde1c571 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 325b5e41-49de-405e-b785-7f72cdc3fb5f kind: Event lastTimestamp: "2026-07-07T15:19:44Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-07T15:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:44Z" name: isvc-autogluon-v2.18c00b08cf057041 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: a542cd19-5788-4787-af34-7729298baf23 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:10:11.529337Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-757779fc66-g2crp to ip-10-0-140-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a834f7db1c0 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: f05647d8-d8c1-4f4e-b3be-8fe47255f37d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a8368f6e06b namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 0414143e-c9ae-434d-91a5-1a407589ee15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a836a888d1d namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: b5d8f4b0-e299-4510-b433-c6fb3d9fe470 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:12Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a836d91296e namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 5272f22d-fae2-4396-9793-47e899f3128c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:12Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a836e032658 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 28516b27-3e21-4758-ae3a-b0b09dba403e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:15Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a84583f61f8 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: 2dd41d86-0ab3-428a-88bd-c28dd710bb06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845b680609 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 2c71b490-4c86-4413-8ecc-eddfd2c4a90f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845bdc246a namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: ac5c93bb-776b-4a15-8342-1c32bbcfae98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845be6fd2d namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: e13c4ce6-d7dc-4be8-950a-111143e01aff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845f42f4f6 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 143268ab-d70e-4715-a958-61941aeb406b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845fb49b79 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: f6ef23cf-dd5b-45d8-8832-d65bcc1d2a97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a845fbeb75a namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: b56aa89d-5e97-4440-b0ba-4da288a03bd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a846275c363 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 5c5daf51-db63-4a88-b75c-473007d0ff6c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:10:16Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:16Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a8462e77933 namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: b67e10df-37fd-476b-b9cf-6dd5e59b5877 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:11:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:53Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a84944707fa namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: a76797fb-cb2f-4685-a87e-3c11dc7420a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:11:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:54Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a84d01465e3 namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: 043eb0d8-c94d-4e74-bb4a-f4dae6b015d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:46Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a997f9b5743 namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 9cf8501e-f74a-40e2-82aa-96ec53463853 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:46Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a997f9db33f namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: a7b3df70-db8e-4947-b6bb-8745a650b0f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:46Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a997fa0a4f3 namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: 6e7fde3d-ebf7-4512-a23b-fa5b9a9fd79e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-g2crp namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 142f6230-386d-460c-9876-12d4fdff3efd kind: Event lastTimestamp: "2026-07-07T15:12:03Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:03Z" name: isvc-logger-predictor-757779fc66-g2crp.18c00a9a00599888 namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: 70297c08-877a-4cdb-a93f-c69e6a26f9b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-757779fc66 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: efc70a2e-99e1-4749-b175-b61afc754c76 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: 'Created pod: isvc-logger-predictor-757779fc66-g2crp' metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger-predictor-757779fc66.18c00a834efaaa3d namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: ef03f7d3-f4bb-4a0b-9fa7-1907005439a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 7e1fca47-feb8-42e3-872f-0f646e5fb6d3 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: Scaled up replica set isvc-logger-predictor-757779fc66 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger-predictor.18c00a834aaa0f31 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 303795e6-ab49-4b4f-97a2-9327ef1caa24 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 22a99309-4d6e-4ba9-a27c-08c132a3eca5 kind: Event lastTimestamp: "2026-07-07T15:10:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:56Z" name: isvc-logger-predictor.18c00a86c98dd00a namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 07e31c30-089b-4edd-af83-a6dd203e3750 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 22a99309-4d6e-4ba9-a27c-08c132a3eca5 kind: Event lastTimestamp: "2026-07-07T15:10:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:56Z" name: isvc-logger-predictor.18c00a86ca1ff120 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 3e906e48-1221-4abe-8976-d4cfa8408531 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 22a99309-4d6e-4ba9-a27c-08c132a3eca5 kind: Event lastTimestamp: "2026-07-07T15:11:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:41Z" name: isvc-logger-predictor.18c00a8a48fb2069 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 4d425f42-a032-4408-95a3-da541d5cac94 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 22a99309-4d6e-4ba9-a27c-08c132a3eca5 kind: Event lastTimestamp: "2026-07-07T15:11:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:41Z" name: isvc-logger-predictor.18c00a8a496b4580 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: b2799374-c786-4b3f-a46c-7e7767333535 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: d006f69e-6993-42e1-8987-6865e8314554 kind: Event lastTimestamp: "2026-07-07T15:11:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:34Z" name: isvc-logger.18c00a834aaaf3dc namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: a9c85642-890b-4b87-a4a7-f8b829b16e98 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: d006f69e-6993-42e1-8987-6865e8314554 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:11Z" name: isvc-logger.18c00a834b3ac0df namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: e6dfea7e-a130-4608-a397-961cc505c26d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: d006f69e-6993-42e1-8987-6865e8314554 kind: Event lastTimestamp: "2026-07-07T15:11:34Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-07T15:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:34Z" name: isvc-logger.18c00a968a1da3ee namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 45b00f8d-bcb0-4392-b692-a1f4ca395478 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:08:26.289019Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p to ip-10-0-140-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6aceade2c6 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6c0fbaac-ceb8-41ec-b166-5699abf90618 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6ae8e91ef8 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 83cf5b0a-4b8d-4e51-bb39-81679d2e29b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6aea9e5a83 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: de19d6dd-2dec-4f54-a1cc-850bac036565 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6aed5b52f0 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: ffd8b249-2fa2-4e38-9df9-64e316e8fdb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6aedd68e8d namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 308ff8d3-905c-4404-b448-a1895c4fe698 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c07c692b2 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: a9058b81-d755-41ca-9413-449fdc086794 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0b0bad6f namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: ead74694-b008-4930-b303-19fbae2dd132 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0b89c150 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 21895eef-e670-4836-9671-10f3adcf8f0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0b968a50 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 038b68e2-8f98-4d71-853f-b7603ee2f462 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0e986bab namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: a907ae1b-b6fe-44c9-800a-22f68403fe8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0f16e19e namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 020c35d8-81da-4100-bd12-e61df6a482b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c0f227fff namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 56de8f51-7aa9-4186-8f26-8e6ccd7d0166 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c124039e4 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 1a051281-5d10-4efc-993a-bd0ede331bd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:08:31Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T15:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:31Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c12bd76e5 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 5b68cc8c-5587-4432-8700-35f2b2fcc27a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c44005bb7 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 7b38ca75-7084-4932-944a-8a87ace02fe3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:08:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a6c800dfd1e namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 2594abc1-a92f-4bec-b26b-14eebe508b69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:01Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a80f881ef16 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: c6d33212-8509-4a77-8c1d-e268e2420dbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:01Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a80f8831e5d namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 8277c34d-46b9-4de5-a94a-11941e96d63b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:01Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a80f88412ea namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: c2d39499-4ae9-4a2c-9ca8-2f400037e31a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 4848a366-b814-4b63-920a-0f3edd01a0e5 kind: Event lastTimestamp: "2026-07-07T15:10:09Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:09Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p.18c00a81b02f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 1ac5c390-ee3a-4841-840c-21de0a77b37d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-684888dfc6 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: b7593f00-3d9f-498c-8849-3e2e2c1375be kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-684888dfc6-bm24p' metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6.18c00a6ace30d09f namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: dcf2132d-f1ee-409b-8424-b05b4dc50e9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: f879582f-42b2-4b44-8539-3776ab1977e0 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-684888dfc6 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom-predictor.18c00a6ac79fb1fb namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 3058d160-75cc-4df3-8656-76ec9f785dcf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: c48fe7bd-c278-4556-983a-a23496609581 kind: Event lastTimestamp: "2026-07-07T15:09:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:11Z" name: isvc-sklearn-batcher-custom-predictor.18c00a6e4624f1c9 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 039f9eb0-ae7a-4e1a-903a-944fb5152db4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: c48fe7bd-c278-4556-983a-a23496609581 kind: Event lastTimestamp: "2026-07-07T15:09:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:11Z" name: isvc-sklearn-batcher-custom-predictor.18c00a6e469b4ab6 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 625443fc-1912-4b61-b7c3-e13a2a842a2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:09:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: c48fe7bd-c278-4556-983a-a23496609581 kind: Event lastTimestamp: "2026-07-07T15:09:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:56Z" name: isvc-sklearn-batcher-custom-predictor.18c00a78c43892d3 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 1db49ebf-930c-4990-b000-daffdfe0c779 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:09:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: c48fe7bd-c278-4556-983a-a23496609581 kind: Event lastTimestamp: "2026-07-07T15:09:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:56Z" name: isvc-sklearn-batcher-custom-predictor.18c00a78c4ab49b1 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 548ffb3c-6357-412e-8447-1d04187fcda1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 4fa79914-dccf-4bed-9a9f-8e3536f48121 kind: Event lastTimestamp: "2026-07-07T15:09:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:09:53Z" name: isvc-sklearn-batcher-custom.18c00a6ac7a22d13 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 78ad902c-2419-4e4c-af89-b2a04dd7c724 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 4fa79914-dccf-4bed-9a9f-8e3536f48121 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-custom.18c00a6ac83e1bb9 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: 486bafcf-5e43-4210-b641-10875d91a9f1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 4fa79914-dccf-4bed-9a9f-8e3536f48121 kind: Event lastTimestamp: "2026-07-07T15:09:53Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-07T15:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:09:53Z" name: isvc-sklearn-batcher-custom.18c00a7f360a295d namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 3f16e0b4-ef7a-4c5f-af96-55477110f578 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:06:31.359943Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-956c77b89-btfqw to ip-10-0-140-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a500c5f53c9 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 16531cb3-4076-44df-ba6b-a4daed44a4fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a502616f8ed namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: df0595af-b194-4763-bab5-493c4f06c736 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a5027e92ffb namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: d281a354-8ff9-4b20-8ba0-4d3a2527cabf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" in 3.622s (3.622s including waiting). Image size: 301881278 bytes.' metadata: creationTimestamp: "2026-07-07T15:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:35Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a50ffda9376 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: cc3f223a-a1bb-4f1e-a6f9-a679db6d2549 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:35Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a51034b04cf namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: eab7c321-d4eb-4b76-81fc-f9b43b9da9e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:35Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a5103befc97 namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: 79a75ea8-5e92-4b77-a10d-868cccfc53f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:40Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:40Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a5214e7691f namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 53037730-4904-4a13-b126-c9efe20a8a87 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 13.335s (13.335s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-07T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:53Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a552fc650bb namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 65dd3dc5-9301-4a86-8097-84de786a9997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:53Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a5533a4e3a4 namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 643a13fb-8507-46e7-9932-5037d800ed97 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:53Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55341bf630 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: e3108a99-0636-4b99-a805-10c1de0f1451 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:53Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:53Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55342a40c8 namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 1381a37b-0aa0-4dfa-bdb7-755148fd76f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.265s (2.265s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55bb2fc949 namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: fbc3f2b6-288c-416b-af3a-7addb209a9ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55be559abc namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: c4c67806-7388-4ae2-9182-19b29124a7fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55bece8a0d namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 78090603-9d1a-4f31-8eba-76e7a6b9dc0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a55bedb2e7b namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 1dffef28-be98-4ecb-8e73-23f38b78ede7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" in 3.724s (3.724s including waiting). Image size: 239995486 bytes.' metadata: creationTimestamp: "2026-07-07T15:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:59Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a569cdfcaba namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: afc401d8-2434-450a-9996-51433ca751d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:59Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T15:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:59Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a569ffaaed7 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: d8f39ace-d5b2-4ccb-9b4b-50e095fa784f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:06:59Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T15:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:59Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a56a0712b3e namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: b783a6ca-d091-4a6a-b254-83e2a7d261e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a56febda703 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: b0ff7e96-c38d-4075-96fa-41b864de5688 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a56fec64038 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: d9c43e81-ad86-49e8-b111-d5f9580f0d84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a6ace2a1674 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 1ab8e69a-1cb8-46cb-b804-0d4136f7f0aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a6ace2a5eb7 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: da5f98c2-10a2-4b55-b594-20b7dc21141a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a6ace2afaf0 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 8f385ee2-9c93-469a-9006-396d609ad33d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-btfqw namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 713fd33d-600f-4b2d-bcc8-adb3855efe06 kind: Event lastTimestamp: "2026-07-07T15:08:50Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:50Z" name: isvc-sklearn-batcher-predictor-956c77b89-btfqw.18c00a6bb75b8827 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: b24f6fd8-b4ac-461d-8e68-f212e2b071a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-956c77b89 namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: 9f29ec9c-414c-4911-8625-91bfda988ccd kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: 'Created pod: isvc-sklearn-batcher-predictor-956c77b89-btfqw' metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher-predictor-956c77b89.18c00a500aee8ce1 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 379e479f-77ae-4f1d-9338-e1b22dd495d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 026c6056-6d9b-4eec-920f-40a0c2241613 kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-956c77b89 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher-predictor.18c00a4ff95e09d8 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 3ce1ac3d-64e0-40bb-84a9-edf513e03ff6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 117bc304-bb35-4289-9fb7-0a92e88e9b90 kind: Event lastTimestamp: "2026-07-07T15:07:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:16Z" name: isvc-sklearn-batcher-predictor.18c00a537fe52e71 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 801d1a86-819c-46eb-93d8-8c3e6e0525c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 117bc304-bb35-4289-9fb7-0a92e88e9b90 kind: Event lastTimestamp: "2026-07-07T15:07:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:16Z" name: isvc-sklearn-batcher-predictor.18c00a5380e459f5 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 8d34466b-87f9-4b7f-b58a-76cfa1436963 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 117bc304-bb35-4289-9fb7-0a92e88e9b90 kind: Event lastTimestamp: "2026-07-07T15:08:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:16Z" name: isvc-sklearn-batcher-predictor.18c00a5dfe882c36 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 4572bdcb-8213-4f3d-a2d5-22518d6d348a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 117bc304-bb35-4289-9fb7-0a92e88e9b90 kind: Event lastTimestamp: "2026-07-07T15:08:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:16Z" name: isvc-sklearn-batcher-predictor.18c00a5dfef68a8d namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 10eb1038-6f71-49a3-b098-841af9538b6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 0a7c9e6f-947f-4db6-a724-05ef2d8ef06e kind: Event lastTimestamp: "2026-07-07T15:08:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:12Z" name: isvc-sklearn-batcher.18c00a4ffff3567d namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 7bf6edc0-01ce-4f3f-9a46-11866a78432f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 0a7c9e6f-947f-4db6-a724-05ef2d8ef06e kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher.18c00a5000ea1426 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 46e030a0-81b9-41a8-938d-8df9cbf77643 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 0a7c9e6f-947f-4db6-a724-05ef2d8ef06e kind: Event lastTimestamp: "2026-07-07T15:06:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:31Z" name: isvc-sklearn-batcher.18c00a5000ea762d namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: fb2b352f-6d2e-4ac8-b82d-fbfe9cdded38 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 0a7c9e6f-947f-4db6-a724-05ef2d8ef06e kind: Event lastTimestamp: "2026-07-07T15:08:12Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-07T15:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:12Z" name: isvc-sklearn-batcher.18c00a6792b43425 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 57b30825-14e4-4aac-8948-024f5e0d1cba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:10:01.455950Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-c78d7 to ip-10-0-140-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a80f712057f namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 3238fc25-7efc-4373-abdc-244c137cad6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-785fd845c9-w978t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8110c1cc3f namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 69c20425-a631-4a2a-9f6e-df2281c8cafc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a811237e5b6 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 3988bd52-665c-4b56-85b1-73b711cc05d9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.626s (1.626s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8173259949 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 6022c285-3122-4b78-b624-d42b2dc539b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8176496974 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 5501b71a-e40e-4ef4-85f5-8468f8aeb746 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8176b89925 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 87639558-95b5-4842-8aa0-c18070e3f035 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8176c20642 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 0bde4339-e5ab-4313-b2e3-12360ca05edc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a8179d34b0c namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 0c7cbd93-3b50-48c8-9d83-699d125b01ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:10:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:03Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a817a670033 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 771628a2-f5d6-418e-ad97-bbad2269a71f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:46Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a99824b6d6a namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: d56b3280-c36f-4585-bbb3-1a6fac069836 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-c78d7 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ea5c3a1b-4411-4120-a9d4-35a73eae0746 kind: Event lastTimestamp: "2026-07-07T15:11:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:46Z" name: message-dumper-predictor-6575bcfdfd-c78d7.18c00a99824bfef8 namespace: kserve-ci-e2e-test resourceVersion: "17131" uid: a4d426fe-b6dd-4804-bd44-9c6d020937ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-158.ec2.internal source: component: kubelet host: ip-10-0-140-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: f6893c9f-2be4-4b70-b1e5-9b153249dcaa kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-c78d7' metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper-predictor-6575bcfdfd.18c00a80f69ac6c5 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 90d9d609-eec2-4e4f-9ff1-db47e7470c6f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 6da0a1c3-f5a0-4b95-88c4-6b92b25bc8da kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper-predictor.18c00a80f2848d80 namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: df326017-b55a-4c78-997c-389cf9b32410 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 99394193-faf3-4585-acde-21ebbcef7a9a kind: Event lastTimestamp: "2026-07-07T15:10:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:31Z" name: message-dumper-predictor.18c00a847107c57a namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: bce8e749-4b7c-4791-b9e5-c56a6718a093 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 99394193-faf3-4585-acde-21ebbcef7a9a kind: Event lastTimestamp: "2026-07-07T15:10:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:31Z" name: message-dumper-predictor.18c00a847173d94b namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: ee80e62c-b53c-4a92-b078-246beb76339b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: e9fd96e6-eaf0-431a-94de-9dd1fa492eb2 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:11Z" name: message-dumper.18c00a80f282e367 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: cee208bd-f3cd-44b8-8792-1a8394f9beb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: e9fd96e6-eaf0-431a-94de-9dd1fa492eb2 kind: Event lastTimestamp: "2026-07-07T15:10:01Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:01Z" name: message-dumper.18c00a80f31e602a namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 946628b1-249f-4138-9233-857bfe8dd427 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: e9fd96e6-eaf0-431a-94de-9dd1fa492eb2 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:11Z" name: message-dumper.18c00a8330c640a6 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 7146ce77-d280-4fb0-b6d4-7abda0a437ab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "23205"