--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:40:01.500833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff70ad7783b namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: f432e963-6490-449e-900e-86812b20bf45 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-ts-v1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff718e4fda2 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d309f335-79f8-4ff0-8a60-e0baaa14bc8d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:02Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:40:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff748949102 namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 6c89f2b4-f8af-451a-b966-3eb1ec63549a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff749f8220f namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 98420d51-80e0-46b0-a846-4b2af283ae7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff74cb955cd namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: db8ad436-dc45-43c1-8db0-b0a5c9100f55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff74d313fa2 namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 5615ddb4-e89a-452a-bddc-d1a0349e5cce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff968e91319 namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: aee78f5c-b1a5-4467-9114-f5e9c9a46395 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff96bfd72fa namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 635073b7-8cce-4fa7-a3c5-87ac964bb4c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff96c778a57 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 0f419408-6f38-4ff7-beec-4c194fb8bf51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff96c822e39 namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: 687f9403-c527-4716-863d-5cb6055e9282 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff96fae8dc1 namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 3688b8ae-d4ec-4e8f-b98c-da2b3b6f91cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff9702247c7 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 554461fa-f5df-473f-bd4b-0ecb2c93dc1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:18Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ff9a53ad834 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 766a18e2-da86-4a03-af9f-e0808edf6c75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:31Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ffe0a4ed593 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: c6415664-37a2-4735-80de-0f004315f759 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 4a441638-1e97-4917-b1ff-b60ab487772f kind: Event lastTimestamp: "2026-07-02T14:40:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:31Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b.18be7ffe0a4f6cad namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: 9e141ccf-5e70-4c53-887e-3d6f263db3a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 35dc084c-fd6a-4090-8481-c58e60c14c0e kind: Event lastTimestamp: "2026-07-02T14:40:01Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-n4c4b' metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18be7ff70a0d9d6b namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 87d29965-9ac1-485a-9a0f-ae111ae5071b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 33b39667-a8b6-4f19-bbe6-349ff312743d kind: Event lastTimestamp: "2026-07-02T14:40:01Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1-predictor.18be7ff704de2468 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: 1c30d77a-3fa6-419d-9297-927d21f9a3ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:40:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: e77bf786-4682-4bdd-92e8-d6d069aa2930 kind: Event lastTimestamp: "2026-07-02T14:40:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:40:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:31Z" name: isvc-autogluon-ts-v1-predictor.18be7ffa83d8dd1b namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 787cee4c-94bd-4c5f-afbc-bfb659a7a386 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:40:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: e77bf786-4682-4bdd-92e8-d6d069aa2930 kind: Event lastTimestamp: "2026-07-02T14:40:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:40:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:31Z" name: isvc-autogluon-ts-v1-predictor.18be7ffa845a9994 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: f3058fe4-950c-4b01-88d4-316c975038b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 1b5b72dd-accd-4420-a01b-4de96116b40d kind: Event lastTimestamp: "2026-07-02T14:40:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:28Z" name: isvc-autogluon-ts-v1.18be7ff704ebbc52 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: f2540b0f-a092-49b9-b3b0-b389bf995e94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 1b5b72dd-accd-4420-a01b-4de96116b40d kind: Event lastTimestamp: "2026-07-02T14:40:01Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1.18be7ff7056f8944 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: c10294f0-67cc-4fd5-8e94-9eb86e58d05a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 1b5b72dd-accd-4420-a01b-4de96116b40d kind: Event lastTimestamp: "2026-07-02T14:40:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:01Z" name: isvc-autogluon-ts-v1.18be7ff7056fe2a9 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 6e847ea0-6153-46d9-a9d5-a72f08675e25 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: 1b5b72dd-accd-4420-a01b-4de96116b40d kind: Event lastTimestamp: "2026-07-02T14:40:28Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-02T14:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:28Z" name: isvc-autogluon-ts-v1.18be7ffd64c24adc namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 54f17dd6-c86a-4961-b76e-8f83ad5f122c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:31:40.341847Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17796" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f825b7035bb namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: eaf0b2a8-1324-4742-88f4-56914db40678 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17822" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f8275ca1905 namespace: kserve-ci-e2e-test resourceVersion: "17847" uid: 1eb1332a-ca38-4146-b7a0-71a3f83136d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f82776d42d2 namespace: kserve-ci-e2e-test resourceVersion: "17849" uid: c5f7cf60-7703-44cb-b824-1d580a72f164 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f827a1fe639 namespace: kserve-ci-e2e-test resourceVersion: "17850" uid: e85eacd5-72bb-4afc-93ef-9b3e49d7571c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f827a9a69e1 namespace: kserve-ci-e2e-test resourceVersion: "17851" uid: ae2522fe-30e8-40c5-bbd3-8fdeede511cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:31:46Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-02T14:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:46Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7f83c61d0fb6 namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: 02fdd63b-d82f-465c-8591-578341c209ff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:06Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m20.378s (2m20.378s including waiting). Image size: 8939170586 bytes.' metadata: creationTimestamp: "2026-07-02T14:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa475527f03 namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: fdc02605-0953-4767-95af-a02d975bf87a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa47b80617f namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: a091f577-d18d-4ac8-be42-ab40e2c9af98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa47cce22a1 namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: ba1ed7fe-bbc5-41df-a77f-f18f801b8b88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa47cdb1e85 namespace: kserve-ci-e2e-test resourceVersion: "18880" uid: ab03fd39-6c1f-4e2a-9e69-32d96363fba4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:07Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa48191d4c9 namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: da155cf3-61e4-44d2-bf62-f320fc51c341 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:07Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa48284ad9e namespace: kserve-ci-e2e-test resourceVersion: "18882" uid: 5ca63f11-7edd-4a2f-8244-9320b0b4af50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa486c13a86 namespace: kserve-ci-e2e-test resourceVersion: "18934" uid: 469d84fc-5a00-439e-800b-4493f8ddbb0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:30Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa9f002a6cf namespace: kserve-ci-e2e-test resourceVersion: "19076" uid: 05d375ef-6edf-48e9-832c-1994962973bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 51d42ab1-c9d6-4b22-9654-9c973dcc4692 kind: Event lastTimestamp: "2026-07-02T14:34:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:30Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw.18be7fa9f00323f1 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: bfdd84cd-c886-4a65-b210-7b8a74258d4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17784" uid: 5b95ad34-fa04-4808-b307-ed08aca10d59 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-d2dgw' metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18be7f825a4237d9 namespace: kserve-ci-e2e-test resourceVersion: "17801" uid: 8f0427aa-de60-486b-9816-47dae9013fa1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 1ea06b2e-0c20-4fed-8108-da180a374aca kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1-predictor.18be7f824fcefb24 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 2beb5a12-c68e-4d5c-b5a6-aa898ed73ccc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:31:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17783" uid: f0680ad1-9cf3-4944-b1ab-dca651501875 kind: Event lastTimestamp: "2026-07-02T14:34:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:31:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:25Z" name: isvc-autogluon-v1-predictor.18be7f85ce4df26b namespace: kserve-ci-e2e-test resourceVersion: "19022" uid: 6982babf-90a9-4e3c-9ff1-3f036951a610 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:31:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17783" uid: f0680ad1-9cf3-4944-b1ab-dca651501875 kind: Event lastTimestamp: "2026-07-02T14:34:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:31:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:25Z" name: isvc-autogluon-v1-predictor.18be7f85cec764d6 namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: bafd08bc-dbdc-4055-8a06-94220910e120 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 94894cc7-d172-4123-8000-e29864ae85ff kind: Event lastTimestamp: "2026-07-02T14:34:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:34:23Z" name: isvc-autogluon-v1.18be7f824fc9ea10 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: ec48f231-05a2-4515-a10b-ebd845ba22f6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 94894cc7-d172-4123-8000-e29864ae85ff kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1.18be7f8250832f61 namespace: kserve-ci-e2e-test resourceVersion: "17788" uid: ec6a9adc-afe4-42da-82ea-72cd14135c91 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 94894cc7-d172-4123-8000-e29864ae85ff kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:31:40Z" name: isvc-autogluon-v1.18be7f8250838293 namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: 8d183305-3bd7-4ba2-8d52-4894213dc2d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 94894cc7-d172-4123-8000-e29864ae85ff kind: Event lastTimestamp: "2026-07-02T14:34:23Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-02T14:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:34:23Z" name: isvc-autogluon-v1.18be7fa8462b15ac namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: 0b6d709b-cbde-47c7-b89c-b8ca9313d0c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:16.214417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fde8748586a namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 9efda3e8-d882-40a4-abce-bc879a696f4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdea11d458d namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 60e54f6a-e5e4-4205-92b6-72b2599aab42 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdea299cb7b namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 9ef60d37-1b62-4c01-8565-bb9cc462cea3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdea516f303 namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 09dc2273-9790-4f71-b9c9-db82e5f4337e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdea58ab0c9 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 3fc662b8-1448-4bf0-ad6b-2ae884fa5e63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdfec01d274 namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 1e85e724-1dd8-4403-82a7-6b1ce6af1a4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdfef636845 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 6a6f80a8-cc55-4150-b88a-0fd245487c89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdfefdbeaef namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 4c80bf95-376e-4b32-8c9f-424af2e55194 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdfefe7f68c namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 2bac20f6-e092-42eb-b34e-5423037cd6c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdff31ac6b8 namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 7ffb4358-b089-482b-a990-a2b00e5202cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7fdff3961f26 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 78e6ea65-ac39-4c7c-9ad1-48c7d2fc5f2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:39:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:56Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7ff5d70a0aaf namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 3ab21ffb-67ba-4aef-af8e-521da0347793 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 184b7b69-e986-4349-ac93-4d8306f2edb2 kind: Event lastTimestamp: "2026-07-02T14:39:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:56Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp.18be7ff5d70ab94d namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: d01f755e-d06b-4b2e-9a77-2ccf3d6c6fd9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 013026db-e2d3-4dc6-9b7c-6bb5fe385ed6 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fghbp' metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18be7fde86d902d0 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 298b1ed4-3f1a-4e10-a76f-a90125150984 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 66523fdb-ae14-4728-8f48-06a9abd8e0c6 kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash-predictor.18be7fde822e155f namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 7e7ce2e2-63ff-4bef-81c1-eddb71d0f08d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 3348045b-2731-449b-aafc-30164ffe67c0 kind: Event lastTimestamp: "2026-07-02T14:38:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:31Z" name: isvc-autogluon-v2-noslash-predictor.18be7fe200995d81 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 12fd391a-9abd-4a28-b90c-216be8b47c7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 3348045b-2731-449b-aafc-30164ffe67c0 kind: Event lastTimestamp: "2026-07-02T14:38:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:31Z" name: isvc-autogluon-v2-noslash-predictor.18be7fe200fba799 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 9e6337ed-8bf6-41dd-a6f9-5768ecc39eac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 3348045b-2731-449b-aafc-30164ffe67c0 kind: Event lastTimestamp: "2026-07-02T14:39:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:46Z" name: isvc-autogluon-v2-noslash-predictor.18be7fe58008d2e7 namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 34c1defe-9fbf-46ab-83f8-ba8f24f5ec54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 3348045b-2731-449b-aafc-30164ffe67c0 kind: Event lastTimestamp: "2026-07-02T14:39:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:46Z" name: isvc-autogluon-v2-noslash-predictor.18be7fe5807b4552 namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: f6b04067-30e4-431a-a779-97e9a2304e57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 284b4263-8a30-4c35-9958-def5b9e5776b kind: Event lastTimestamp: "2026-07-02T14:39:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:52Z" name: isvc-autogluon-v2-noslash.18be7fde82375c6b namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: b100f74b-0702-4661-a997-b871c3647f01 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 284b4263-8a30-4c35-9958-def5b9e5776b kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash.18be7fde82bbc617 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 2460de95-ca6c-46b1-84b4-438955c37e06 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 284b4263-8a30-4c35-9958-def5b9e5776b kind: Event lastTimestamp: "2026-07-02T14:38:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:16Z" name: isvc-autogluon-v2-noslash.18be7fde82bc50ab namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: c4571869-fa1e-4bee-b3d8-556da143af33 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 284b4263-8a30-4c35-9958-def5b9e5776b kind: Event lastTimestamp: "2026-07-02T14:39:52Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-02T14:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:52Z" name: isvc-autogluon-v2-noslash.18be7ff4f66710a8 namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 9adfce30-8c46-4171-9c59-37061e5acb76 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:34:35.617795Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19142" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-mgc9f to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:34:35Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fab2ab3073e namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: 6384de98-b93a-4b98-a469-4d6d46a41017 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19154" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:36Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:34:36Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fab4532642e namespace: kserve-ci-e2e-test resourceVersion: "19162" uid: a1b1349b-b7f4-45db-9438-d4cf58525220 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:36Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fab47070e52 namespace: kserve-ci-e2e-test resourceVersion: "19166" uid: fc25aa5a-80d4-4e1f-aa1e-117826836713 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:36Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fab4bd02ffd namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: 0ab74d27-067c-4e59-a85c-00f8b777638c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:36Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fab4cb4ff22 namespace: kserve-ci-e2e-test resourceVersion: "19168" uid: 2491e93d-3112-472c-b4b2-298e99043e6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac79e7e9ca namespace: kserve-ci-e2e-test resourceVersion: "19209" uid: e8edee95-0339-4995-be4c-26ffc377cbea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac80940612 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: 1a6a2e9b-0c76-4d65-813f-9bc146becabe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac81df6ab6 namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: c64874b9-a181-486c-bb1d-16a300fa789f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac81eabe43 namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: 9e1b4761-2a93-4861-b2f0-251f4165116e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac87e152ca namespace: kserve-ci-e2e-test resourceVersion: "19219" uid: bebe0897-76a1-407c-b83d-d1c662a7dbfb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:34:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:34:41Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fac894d3cd3 namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: 25ccc45e-4a96-42de-9cbe-83215a5cfe61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:36:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:15Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fc27b6cc9e3 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: ef87bfac-e790-4516-aa37-f0d4c8a8a4b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mgc9f namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: a9eaf80e-5300-4198-956e-0f00b1984d04 kind: Event lastTimestamp: "2026-07-02T14:36:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:15Z" name: isvc-autogluon-v2-predictor-67b849d456-mgc9f.18be7fc27b6e991b namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 0fe30039-9413-4496-b899-137c85f871f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "19131" uid: b2ee2185-0318-4255-9ace-2c91a5756424 kind: Event lastTimestamp: "2026-07-02T14:34:35Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-mgc9f' metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:35Z" name: isvc-autogluon-v2-predictor-67b849d456.18be7fab2a4419ac namespace: kserve-ci-e2e-test resourceVersion: "19143" uid: a72989da-3fc4-44a2-ad95-305ae9611a4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:34:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: 593fc969-d29b-4540-8f6c-1367037440d3 kind: Event lastTimestamp: "2026-07-02T14:34:35Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:35Z" name: isvc-autogluon-v2-predictor.18be7fab25a83495 namespace: kserve-ci-e2e-test resourceVersion: "19132" uid: 259a1bb3-23ae-4d16-823e-873e4c19571c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:34:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: b9df242d-593c-497a-ae50-10689747b064 kind: Event lastTimestamp: "2026-07-02T14:35:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:34:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:05Z" name: isvc-autogluon-v2-predictor.18be7faea497e1f7 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 1b4cae38-a6e1-4cce-be2d-a6770fc3ce40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:34:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: b9df242d-593c-497a-ae50-10689747b064 kind: Event lastTimestamp: "2026-07-02T14:35:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:34:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:05Z" name: isvc-autogluon-v2-predictor.18be7faea50ad8ed namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: a2c4fa22-4ea0-4d2d-99bf-55d47ecc117e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:35:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: b9df242d-593c-497a-ae50-10689747b064 kind: Event lastTimestamp: "2026-07-02T14:36:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:05Z" name: isvc-autogluon-v2-predictor.18be7fb5a3baddbc namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: 3390f404-81f1-4dd0-b3d2-5ca255142c7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:35:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: b9df242d-593c-497a-ae50-10689747b064 kind: Event lastTimestamp: "2026-07-02T14:36:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:05Z" name: isvc-autogluon-v2-predictor.18be7fb5a426ca43 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 44418884-19c7-4827-b01a-2f5968c8a870 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:36:20.936731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:36:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc3b0325d65 namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 2cf662b5-2688-4c3a-b6e7-bae16f7625f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:21Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:36:21Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc3c9cb3a86 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: dec8fb3a-4ca5-41cd-ba90-e138449abb19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:21Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc3cb3a42c1 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: a15500e5-a890-4109-8039-a98854ef6bfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:21Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc3cdcc8913 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 00df6236-9ae7-4f0e-9a4c-484cfee45142 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:21Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc3ce334976 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: dfca675d-42bf-428d-b19a-964134b378dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50759b0f3 namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 028120a4-62b3-4b35-91b3-3eb5c9c1ed55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50a10a1e4 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 9294fa00-b243-44a5-9a71-4002b6486494 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50a772e94 namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 8ca504b4-b569-49ee-87cc-0e75b14fec4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50a80d4ce namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: 20468743-da88-42f0-ac23-be8a0a14866e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50eabe425 namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: da9c3979-a987-4300-9aa3-b2e4edb86d2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fc50f0f79c8 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: c86ec883-c2b2-4a9c-95fe-001daf8a4086 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:38:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fdd57c3db92 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: d1538d29-a1f6-4004-9469-0a5bb4ed348c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 789ef6e7-73f7-47d4-89d8-19680aa71978 kind: Event lastTimestamp: "2026-07-02T14:38:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb.18be7fdd57c4ef96 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: cd1e4e70-7fd0-4d40-8c89-6b53819d6f87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: 44c4ba06-3c4a-4321-b48d-054db5ddc03b kind: Event lastTimestamp: "2026-07-02T14:36:20Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-xb8vb' metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18be7fc3afbb8af3 namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1f617839-e644-42a9-a305-fa03f15f3174 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: d88aee7e-b494-484c-979e-d9d6d449aa9b kind: Event lastTimestamp: "2026-07-02T14:36:20Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:20Z" name: isvc-autogluon-v2-variants-predictor.18be7fc3aab3b1a2 namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 07501a5c-f7c3-49b1-97b2-7f57221f3791 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 40c67e1c-6b1a-416c-92c0-ba4bc3f169b4 kind: Event lastTimestamp: "2026-07-02T14:36:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:50Z" name: isvc-autogluon-v2-variants-predictor.18be7fc729685cfd namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 035c799f-5c79-4ac7-b138-89847bea87fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 40c67e1c-6b1a-416c-92c0-ba4bc3f169b4 kind: Event lastTimestamp: "2026-07-02T14:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:50Z" name: isvc-autogluon-v2-variants-predictor.18be7fc729afdb35 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 96cbc7f9-df96-47e1-ba5a-2f822e9770cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 40c67e1c-6b1a-416c-92c0-ba4bc3f169b4 kind: Event lastTimestamp: "2026-07-02T14:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:05Z" name: isvc-autogluon-v2-variants-predictor.18be7fce27a2aead namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 2e12143a-8a43-4225-9ee8-157e0e7621e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 40c67e1c-6b1a-416c-92c0-ba4bc3f169b4 kind: Event lastTimestamp: "2026-07-02T14:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:05Z" name: isvc-autogluon-v2-variants-predictor.18be7fce28140019 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 367d4629-78c0-423a-955e-3615443e600a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19938" uid: c47429b1-89c4-454a-a0fc-8fec6d767267 kind: Event lastTimestamp: "2026-07-02T14:38:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:04Z" name: isvc-autogluon-v2-variants.18be7fc3aafa227c namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: d40868fc-76b0-446e-9948-ef5df617a988 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19938" uid: c47429b1-89c4-454a-a0fc-8fec6d767267 kind: Event lastTimestamp: "2026-07-02T14:36:20Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:20Z" name: isvc-autogluon-v2-variants.18be7fc3ab5a030a namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: debdc4ec-3630-4a9a-b6da-ab24990e7811 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19938" uid: c47429b1-89c4-454a-a0fc-8fec6d767267 kind: Event lastTimestamp: "2026-07-02T14:36:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:20Z" name: isvc-autogluon-v2-variants.18be7fc3ab5a68db namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 56b9cf4b-5110-4f9d-9dd4-c32b46c3c901 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: c47429b1-89c4-454a-a0fc-8fec6d767267 kind: Event lastTimestamp: "2026-07-02T14:38:04Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-02T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:04Z" name: isvc-autogluon-v2-variants.18be7fdbd1190c06 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 806c65b7-ba36-4afa-8318-2c66c2c0649a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:34:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 0e7f7bb4-96a1-4137-b9b9-5aab262ec9c2 kind: Event lastTimestamp: "2026-07-02T14:36:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:12Z" name: isvc-autogluon-v2.18be7fab25b3ea40 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: ba20d684-e074-4a31-b847-e6d269a28d9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:34:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 0e7f7bb4-96a1-4137-b9b9-5aab262ec9c2 kind: Event lastTimestamp: "2026-07-02T14:34:35Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:34:35Z" name: isvc-autogluon-v2.18be7fab261b7674 namespace: kserve-ci-e2e-test resourceVersion: "19153" uid: 358ab36f-3307-402a-9118-1bcb1d6556fe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:34:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 0e7f7bb4-96a1-4137-b9b9-5aab262ec9c2 kind: Event lastTimestamp: "2026-07-02T14:34:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:34:35Z" name: isvc-autogluon-v2.18be7fab261bdbd2 namespace: kserve-ci-e2e-test resourceVersion: "19155" uid: f0fa212d-5bf4-48f2-b0ad-f8edc9c9cb3e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0e7f7bb4-96a1-4137-b9b9-5aab262ec9c2 kind: Event lastTimestamp: "2026-07-02T14:36:12Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-02T14:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:12Z" name: isvc-autogluon-v2.18be7fc1bd01341a namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: c866eaef-826b-45ea-8460-70eef7eac00b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:30:04.807674Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-756bf5f9fb-28ssz to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:30:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:30:04Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c1d288786 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 1811ec05-f774-4cac-8b45-8dcf7fd3a46b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:05Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c2953ac02 namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 8b2ff0b2-b8ad-4b95-aed9-36793f21c9d8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17031" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:05Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:30:05Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c5b0609e0 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: 4f419ee4-dfd1-4a25-86dd-297df1893514 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:05Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c5c9ed03a namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 7d697d0e-2cf4-4703-9dc9-d1f1c03b6991 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:05Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c613ab785 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: ee0d9bec-9c82-440f-bbb6-f003f4eb6bc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:05Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6c62094c4f namespace: kserve-ci-e2e-test resourceVersion: "17051" uid: 3ca91302-5c5a-4bec-80a3-b9dad832a605 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:09Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d524ec8c6 namespace: kserve-ci-e2e-test resourceVersion: "17085" uid: 4207b763-2718-4577-b511-321fbdc1c08a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5641d497 namespace: kserve-ci-e2e-test resourceVersion: "17087" uid: ebd4cccf-1555-4b21-93ea-2ff68d4c44c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d56d79343 namespace: kserve-ci-e2e-test resourceVersion: "17090" uid: a134ab51-69ae-42a6-a630-b473221a261b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d56e25ead namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 0d7f6bbc-79a3-48f5-99da-c8465ca8181b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5a123253 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: b6c33db6-e9f5-4870-8344-11cf9e492067 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5a8cbc8e namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 7cfb307c-2f3c-4887-b50f-4f74b4e389a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5a978152 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 1ce0b036-2925-4494-a4a0-17895399fd2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5deac547 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 3c70d5f7-5b62-4cc3-9ff6-57a2913a4d20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:30:10Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:10Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d5e66294d namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 04f28b5f-55ca-4515-b80e-098b460aa0fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:41Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6d8e5a54cf namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: 0dad2503-35c8-4e4e-858f-7f244271439a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:41Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f6dca34e866 namespace: kserve-ci-e2e-test resourceVersion: "17865" uid: 7c3e3b04-ddfd-4974-8b3a-815fc142a682 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f826195a952 namespace: kserve-ci-e2e-test resourceVersion: "17833" uid: 6bcd399e-6b23-47c0-b8e7-7f575db7d54c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f826195f07f namespace: kserve-ci-e2e-test resourceVersion: "17836" uid: c850e283-0132-4d79-b0d7-51d982fa28c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f8261963134 namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: aea7f92f-3a9b-4bdd-887a-a36f9eef63d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-756bf5f9fb-28ssz namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 78697d7d-fbc6-4f1b-8754-f028b990e717 kind: Event lastTimestamp: "2026-07-02T14:31:43Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:43Z" name: isvc-logger-predictor-756bf5f9fb-28ssz.18be7f82fa761994 namespace: kserve-ci-e2e-test resourceVersion: "17874" uid: 73fe9273-938f-4eee-8cea-868683237870 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-756bf5f9fb namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: 45a63fa7-a241-4ebb-a6e2-6f1349d72f14 kind: Event lastTimestamp: "2026-07-02T14:30:04Z" message: 'Created pod: isvc-logger-predictor-756bf5f9fb-28ssz' metadata: creationTimestamp: "2026-07-02T14:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:04Z" name: isvc-logger-predictor-756bf5f9fb.18be7f6c1cade301 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 82e2fd08-375d-4d7b-9e09-a4ec066e47f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 27749118-8f32-45f1-9886-804e369b11fe kind: Event lastTimestamp: "2026-07-02T14:30:04Z" message: Scaled up replica set isvc-logger-predictor-756bf5f9fb from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:04Z" name: isvc-logger-predictor.18be7f6c18b1ffad namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: 21b5376b-a071-416e-a145-23151bfd369a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: 3fe3040e-b3d4-4fdb-9237-0ffe262f409a kind: Event lastTimestamp: "2026-07-02T14:30:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:34Z" name: isvc-logger-predictor.18be7f6f974e452d namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 83853c24-5703-4606-b6eb-59874293c05e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: 3fe3040e-b3d4-4fdb-9237-0ffe262f409a kind: Event lastTimestamp: "2026-07-02T14:30:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:34Z" name: isvc-logger-predictor.18be7f6f97c2794b namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 3cf39450-d2c0-49cb-8e31-8683c1d95a3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:30:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 3fe3040e-b3d4-4fdb-9237-0ffe262f409a kind: Event lastTimestamp: "2026-07-02T14:31:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:30:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:34Z" name: isvc-logger-predictor.18be7f7695215f5a namespace: kserve-ci-e2e-test resourceVersion: "17730" uid: d78f75f7-08bb-4d58-ab84-aa5b882bc89a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:30:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 3fe3040e-b3d4-4fdb-9237-0ffe262f409a kind: Event lastTimestamp: "2026-07-02T14:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:30:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:34Z" name: isvc-logger-predictor.18be7f7695918ae7 namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: 0e77cd05-640c-408a-ba96-b6c024819225 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 5df149d5-61c5-4dc8-b364-e95250a4cc64 kind: Event lastTimestamp: "2026-07-02T14:31:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:31:31Z" name: isvc-logger.18be7f6c18d48e3d namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: e2ac053b-f4e2-420b-9f5a-f424958de402 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 5df149d5-61c5-4dc8-b364-e95250a4cc64 kind: Event lastTimestamp: "2026-07-02T14:30:04Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:04Z" name: isvc-logger.18be7f6c1939aba4 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 81cd0bd7-b905-4e5e-a5f8-bb98e6980d77 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: 5df149d5-61c5-4dc8-b364-e95250a4cc64 kind: Event lastTimestamp: "2026-07-02T14:31:31Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-02T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:31:31Z" name: isvc-logger.18be7f8052469cc3 namespace: kserve-ci-e2e-test resourceVersion: "17706" uid: df8a9242-387b-488c-bea0-443b6b147fc6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:09.556792Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5147ab9a1c namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 958cd2a6-d5a2-4999-9e2c-3533f71321d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-batcher-custom-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5154121043 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 8c60f2bf-d13d-4848-8604-51bb27bb731c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:10Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f518556d87d namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 801c72ee-f614-443a-82f3-d319abf351d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5186ee8c1e namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 008b29e2-d510-44ba-88ff-25d2ad3ddc49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f518a170f9f namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 90208287-fbb0-4d16-9def-d0d725d5d6aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f518a89fef5 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: e5d28037-bdb9-466b-af6f-bb54f6d2e304 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f526d4e99a3 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: d85ac6ab-dff8-4453-9b22-7d3362cb2fec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f527032fa8b namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 3a24ee6e-8bb7-4a11-9e86-8e29eb8c34da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5270abcce0 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 60e2cd77-850b-4b07-a24f-6cdd6b8fae44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5270b6ca54 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: e00681b2-e6af-45bb-af22-ebef6c0d40bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5273f188d6 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: f266ea8d-343c-4df1-85f4-d3138e3b070e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5274698a3a namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 99a69746-27a2-4fcb-b6d9-5b42f6600bc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5274744488 namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 0d47a2aa-c1e6-43b6-a1d5-37d02a5c8aa3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f52779a5ce8 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: fdda09f0-c90d-4213-b0da-567e70d6969a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f5278138029 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 98f793d6-aff8-4e17-81ce-b6fa915b1604 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:29:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:51Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f52a982a542 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 60fb5eaa-8fc2-40af-baa6-150a1c44c354 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:29:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:51Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f52a98d1fd1 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 912c5347-dcb0-4bc9-8146-b6160d4a6ac2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:44Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f67743a36fc namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: c8c1cf86-1de4-4241-a418-5036eb9a77b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:44Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f67743a6f20 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7aec932a-ec60-432e-9936-18097b7dffd9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:44Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f67743adf46 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 8d41d57e-cc63-4046-aa80-324043227605 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 613980ec-8ca3-4796-bee0-688ca252bbf4 kind: Event lastTimestamp: "2026-07-02T14:30:01Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:01Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2.18be7f67d9b7ae9b namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: ae7eaa1e-9230-4c34-a94b-cbeeee9c2fd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 32ff9f20-dc9d-4a74-8b42-b434fe27162a kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb-h9vz2' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-custom-predictor-7dbc5fbddb.18be7f51475622cc namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 2d96037a-7847-482c-8649-527a82e47d05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 88b75cd3-1a29-4c3d-9d4d-27a861ef0ed7 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-7dbc5fbddb from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-custom-predictor.18be7f5141e049cb namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 9c74dc53-ba2d-4b2b-b317-eae6fe65ca23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 89691f43-79af-4247-8ba5-03bd858a5efb kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: isvc-sklearn-batcher-custom-predictor.18be7f54c11eeffb namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 87c18daa-c02d-49b3-9119-17e33ef3fe40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 89691f43-79af-4247-8ba5-03bd858a5efb kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: isvc-sklearn-batcher-custom-predictor.18be7f54c184fa38 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 3b854c6c-e1a7-454e-88a4-1a99e4474bda reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 89691f43-79af-4247-8ba5-03bd858a5efb kind: Event lastTimestamp: "2026-07-02T14:29:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:39Z" name: isvc-sklearn-batcher-custom-predictor.18be7f5bbfe58e9b namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: 4b59a2b3-f023-48d0-9e60-dea0c4c254e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 89691f43-79af-4247-8ba5-03bd858a5efb kind: Event lastTimestamp: "2026-07-02T14:29:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:39Z" name: isvc-sklearn-batcher-custom-predictor.18be7f5bc04c2227 namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: d1e3d95c-df09-40ae-99d5-74612924c63f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 1a8b1e26-2472-42b3-8199-4d1c1b4810ce kind: Event lastTimestamp: "2026-07-02T14:29:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:31Z" name: isvc-sklearn-batcher-custom.18be7f514207e5a9 namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 76a5cb6a-0e41-45e9-904c-a9bf40b86840 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 1a8b1e26-2472-42b3-8199-4d1c1b4810ce kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-custom.18be7f514267c66f namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 11520094-adbd-407f-9f02-0ee85139bd1a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 1a8b1e26-2472-42b3-8199-4d1c1b4810ce kind: Event lastTimestamp: "2026-07-02T14:29:31Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-02T14:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:31Z" name: isvc-sklearn-batcher-custom.18be7f64630c23d3 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: c275c7e6-1016-42e3-91d7-301022d27208 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:26:14.412396Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f36788785df namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 2212d926-703a-435f-b299-9be6d4ded081 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f36925d6b3a namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 231f577e-0c4a-42ff-898f-7c50bb85c746 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3693d7ae56 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 64445534-9c08-46e8-b97e-2a75c631cf6f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" in 4.196s (4.196s including waiting). Image size: 301859831 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:19Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f378dfd927c namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: e786c97d-43b5-4c42-82b7-02c8b0a6bc9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:19Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f37945d66b5 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: c79a16f4-d240-4667-8a3c-bf31deefcfbb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:19Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3795709f4c namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: d37e7315-29a8-4b8f-a5d0-05dacf863ba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:22Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:22Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f38773d4aea namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 953c56ff-5a70-4f9d-9981-853ee904d8a1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:37Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" in 14.246s (14.246s including waiting). Image size: 1555893189 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:37Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3bc86b224d namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 08b5b7e8-159f-4310-9aa6-3cc058f4ea4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:37Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3bcd3366bc namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: ed7d2573-8f29-46af-8e7c-f029531554da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:37Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3bcda59c49 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: acde52c0-a841-447a-97db-d040ed679f85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:37Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3bcdb44f46 namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 09921ca8-150d-4b7c-8732-dcc3532c28ef reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.484s (2.484s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:39Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3c61c8fdcb namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 552a32b2-c421-4043-ada5-7bb499b257bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:39Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3c649bc642 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 19284851-6861-4952-b79a-ef6e1fc2f15e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:39Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3c6500eb22 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: eaa463a7-65c3-4b2e-9a09-da004e6fba5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:39Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" metadata: creationTimestamp: "2026-07-02T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:39Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3c650e284e namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: ac52974f-4a4c-490c-9042-79d0634fb0b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" in 3.087s (3.087s including waiting). Image size: 239994522 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:42Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3d1d1049d5 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 785c7400-2cba-464f-8c5d-3d240809b581 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:42Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T14:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:42Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3d1feb53f9 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 683d37fa-478e-474f-9c7b-af85acc2c5a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:26:42Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T14:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:43Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3d20565e94 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 0c786247-3da8-4541-b1bc-cfb9a053170f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:26Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3d2604b25c namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: d26f4807-2fe9-425a-a434-9597ccb429c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:16Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f3d61dec5e5 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 5cf5b64f-5c7c-4e4f-97d7-65d6ca827f10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f5145c76b0a namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 58c65df8-cb48-4eb9-8cd5-bf9a621e03d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f5145ca5b2d namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: dff65feb-7d2f-4378-886c-ce8d54880187 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:09Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f5145ca8e7f namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 69c2b878-5874-4b34-b56f-1732c2e96fe2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 2e353236-2e86-4549-8aca-abea95514e22 kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:24Z" name: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p.18be7f5256494d57 namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 77a8244d-8634-4a31-a10a-7e7426a8a847 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-645bdbffc9 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: b5cee3c0-aa7e-4bb6-a29f-ef9798f519cf kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: 'Created pod: isvc-sklearn-batcher-predictor-645bdbffc9-q6l5p' metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher-predictor-645bdbffc9.18be7f3678235989 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: d5ee3c0a-7e60-4fb2-b98b-cca3f65f408b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: e60beff0-95f2-438e-8520-523266e4d2b1 kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-645bdbffc9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher-predictor.18be7f3670079372 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 6c7730ec-4f43-4f61-9287-afebe30ba4c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 6c0a553c-fcbb-4f6e-9dce-b8be1b5978ed kind: Event lastTimestamp: "2026-07-02T14:27:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:14Z" name: isvc-sklearn-batcher-predictor.18be7f39f55ad6bd namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 1692661f-8094-4e60-89a3-0e46c749a9a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 6c0a553c-fcbb-4f6e-9dce-b8be1b5978ed kind: Event lastTimestamp: "2026-07-02T14:27:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:14Z" name: isvc-sklearn-batcher-predictor.18be7f39f5c82827 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: e9c9f321-0f7e-43c1-9e80-b953578c2f84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:27:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 6c0a553c-fcbb-4f6e-9dce-b8be1b5978ed kind: Event lastTimestamp: "2026-07-02T14:27:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:59Z" name: isvc-sklearn-batcher-predictor.18be7f47f1a1858e namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 6fca97aa-b8b4-4716-aa50-2aed604e4480 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:27:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 6c0a553c-fcbb-4f6e-9dce-b8be1b5978ed kind: Event lastTimestamp: "2026-07-02T14:27:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:59Z" name: isvc-sklearn-batcher-predictor.18be7f47f1fea12a namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: aed2be70-4c03-4e0e-b7cc-5da486b723a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: d5490960-68f0-4198-beab-b2809e465ecc kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: isvc-sklearn-batcher.18be7f36766f8249 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 5c775ca1-b383-44b2-bcc5-f3d551838f27 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: d5490960-68f0-4198-beab-b2809e465ecc kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher.18be7f3676ce058c namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: c991f341-46fc-49ac-b0e9-817fd61afdff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:26:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: d5490960-68f0-4198-beab-b2809e465ecc kind: Event lastTimestamp: "2026-07-02T14:26:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:14Z" name: isvc-sklearn-batcher.18be7f3676ce4e32 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 1181d069-0ed3-40c6-b2a7-70899fdffbe0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: d5490960-68f0-4198-beab-b2809e465ecc kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-02T14:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: isvc-sklearn-batcher.18be7f4e2e0cb23d namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: fbfdfd5a-1a65-4895-a621-8fc84fa26832 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:29:44.748039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-8hkxs to ip-10-0-138-160.ec2.internal metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f677182af12 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 6afba2a6-f914-4d6c-a4b3-8ae36931bc4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-668644474c-dr5jd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f677c8472f4 namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: 69ae68ba-1c05-44cf-8afa-eb6b29fd159f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:45Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:45Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:29:45Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f67af24a2c1 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: fd7489f9-3f94-4c1f-a699-d2bad8eec4e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:45Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T14:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:45Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f67b0a33ce7 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 95d4e94b-e55a-4fce-9a95-3b37332150b6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.498s (1.498s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f6809f90ed1 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: b81a787c-01a6-425b-9640-b354245b7c17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f680d463593 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: 11195000-c830-4d65-9515-c023edf10e08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f680dbee8d6 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: f5d9a04d-f4bc-4711-abe8-74bc705ba4bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f680dc8ad08 namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 23c8c80e-b6e4-471f-a95a-0cada2d098d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f6811047e44 namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: 6bfa0b3b-94c5-4d98-915b-184998f4f5df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:29:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:47Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f68117ef424 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: fa83edcc-386f-4496-9216-c1fcd9f62ed9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f8263c00971 namespace: kserve-ci-e2e-test resourceVersion: "17839" uid: 0192d51b-0023-4027-918f-fab9bd08a86f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-8hkxs namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 512aab7b-05bb-44db-b212-915d9a67de43 kind: Event lastTimestamp: "2026-07-02T14:31:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:40Z" name: message-dumper-predictor-6575bcfdfd-8hkxs.18be7f8263c09631 namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: adf5a313-fd90-4410-8282-c997fb8f2043 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-160.ec2.internal source: component: kubelet host: ip-10-0-138-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: d1bb08aa-eedd-442a-b2d8-cbed8a66bc80 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-8hkxs' metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper-predictor-6575bcfdfd.18be7f677114831a namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 6ada0ffe-b4da-48a0-869b-567c25c8d0f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 8eef85d6-3e94-4a62-93ed-082cc36412f6 kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper-predictor.18be7f676d4ed845 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: d6f5b945-7534-4041-b350-9c4ca1be093f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:29:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 4d29e3a5-5ab1-4936-9596-8d025fe31c03 kind: Event lastTimestamp: "2026-07-02T14:30:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:29Z" name: message-dumper-predictor.18be7f6aebd21a15 namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: 1fe0befd-b381-4067-82cc-2a49391428fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:29:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 4d29e3a5-5ab1-4936-9596-8d025fe31c03 kind: Event lastTimestamp: "2026-07-02T14:30:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:29Z" name: message-dumper-predictor.18be7f6aec113b62 namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: c8d31988-6c44-4525-a1fd-d2e30cf27e9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: f50a5826-9e7f-457e-b19b-0c0fba6d829e kind: Event lastTimestamp: "2026-07-02T14:29:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:55Z" name: message-dumper.18be7f676d8739cf namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: d4560a2a-1e9b-47d3-9e6a-8f53aa568e20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: f50a5826-9e7f-457e-b19b-0c0fba6d829e kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper.18be7f676e03a599 namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 108be980-7b89-48c4-ba6e-cb9213c7a80a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: f50a5826-9e7f-457e-b19b-0c0fba6d829e kind: Event lastTimestamp: "2026-07-02T14:29:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:44Z" name: message-dumper.18be7f677701179b namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: ebe1353f-44ca-49b4-8f20-2c648ef8da35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: f50a5826-9e7f-457e-b19b-0c0fba6d829e kind: Event lastTimestamp: "2026-07-02T14:29:54Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-02T14:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:54Z" name: message-dumper.18be7f69d42d758c namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 8c8fff22-7388-45b5-a882-83aca685be74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22170"