--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:33:22.588121Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:33:22Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e1e5b5e1ae7 namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: 2584d593-1c63-46e8-855b-f13dacb22314 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:33:23Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e1e7544258b namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 5f930524-68da-4f86-b5cb-a7c416fcea6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:23Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e1e77328bca namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 31580d3d-2a6f-4a0b-92ad-7b5cb32de1ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:23Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e1e7a3880e7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: ba4ff063-bb67-45df-ad45-78afb06d76b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:23Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e1e7ad036f7 namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: 03a50a9f-2fbe-4f74-b629-f502ad0ebb97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e0d205de namespace: kserve-ci-e2e-test resourceVersion: "22640" uid: d7a241cc-d7c4-4863-ba54-36d2f655df25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e3d72b68 namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: 241172c9-cb74-442e-b720-bd07a5c2864e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e4565ca6 namespace: kserve-ci-e2e-test resourceVersion: "22646" uid: 1e995c20-c56c-4683-8e60-cfd6bc6b53be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e4625756 namespace: kserve-ci-e2e-test resourceVersion: "22647" uid: 5f03dcb2-30c2-4e57-9b62-31bc17b009a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e7b78316 namespace: kserve-ci-e2e-test resourceVersion: "22649" uid: a073373e-c432-4bd7-bd46-4f5c8b92121b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e20e836d66a namespace: kserve-ci-e2e-test resourceVersion: "22650" uid: 2f3bb75b-7167-4909-9f3a-8fbb342128cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:33:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:33:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:41Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e2158b918e7 namespace: kserve-ci-e2e-test resourceVersion: "22708" uid: 009e6b18-533c-41c4-8732-177209776104 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:52Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e255a47b8cd namespace: kserve-ci-e2e-test resourceVersion: "22816" uid: 033c26fc-f031-4732-8c92-c9ce53d4dee9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 23a3fa31-cecc-45b9-89cb-dd4efc1518f9 kind: Event lastTimestamp: "2026-07-01T18:33:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:52Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt.18be3e255a481e95 namespace: kserve-ci-e2e-test resourceVersion: "22817" uid: ac4ca7d6-0ca3-4966-9594-f3f476deb725 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: 0d232acd-30e0-4803-996b-f9b66860fa1b kind: Event lastTimestamp: "2026-07-01T18:33:22Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-m5npt' metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:22Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18be3e1e5af32aab namespace: kserve-ci-e2e-test resourceVersion: "22538" uid: 63d0c54e-c778-490e-9a84-7a93dd9e8c3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22525" uid: 8d90c58a-3845-4dfc-ba9a-35b244436b28 kind: Event lastTimestamp: "2026-07-01T18:33:22Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:22Z" name: isvc-autogluon-ts-v1-predictor.18be3e1e5669881e namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: 29432042-bed0-485e-9bb5-949844873526 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:33:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: b35254ae-ad7b-47e6-961d-d292f7280da9 kind: Event lastTimestamp: "2026-07-01T18:33:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:52Z" name: isvc-autogluon-ts-v1-predictor.18be3e21d5a3bab4 namespace: kserve-ci-e2e-test resourceVersion: "22800" uid: 9a654500-4865-46d2-b8d8-3b4d55102f52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:33:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: b35254ae-ad7b-47e6-961d-d292f7280da9 kind: Event lastTimestamp: "2026-07-01T18:33:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:52Z" name: isvc-autogluon-ts-v1-predictor.18be3e21d610dda7 namespace: kserve-ci-e2e-test resourceVersion: "22801" uid: 2bf1eafd-6ac1-4552-ba28-6c2cf1ac85b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:33:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: adbc6e3c-060a-42c7-b9dd-6d5139d944c0 kind: Event lastTimestamp: "2026-07-01T18:33:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:51Z" name: isvc-autogluon-ts-v1.18be3e1e567183b8 namespace: kserve-ci-e2e-test resourceVersion: "22792" uid: a0b37239-052b-4711-824c-a09eb62238b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: adbc6e3c-060a-42c7-b9dd-6d5139d944c0 kind: Event lastTimestamp: "2026-07-01T18:33:22Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:22Z" name: isvc-autogluon-ts-v1.18be3e1e56f23a3f namespace: kserve-ci-e2e-test resourceVersion: "22532" uid: b2ef76d8-000c-480b-b614-94d5a15949da reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: adbc6e3c-060a-42c7-b9dd-6d5139d944c0 kind: Event lastTimestamp: "2026-07-01T18:33:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:22Z" name: isvc-autogluon-ts-v1.18be3e1e56f273b5 namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 10983ec6-008a-44d6-9a09-ba1429bd5d96 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22790" uid: adbc6e3c-060a-42c7-b9dd-6d5139d944c0 kind: Event lastTimestamp: "2026-07-01T18:33:51Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-01T18:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:51Z" name: isvc-autogluon-ts-v1.18be3e251842f42a namespace: kserve-ci-e2e-test resourceVersion: "22791" uid: a9cee2e3-b812-42c3-b809-87fc35acac93 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:23:51.120704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18208" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d994d41a22e namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: e66c904c-b380-414c-a4be-448a4ddecb84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d9967a14248 namespace: kserve-ci-e2e-test resourceVersion: "18248" uid: 8cdff378-043b-4273-9f6a-3674afede4cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d9968f4aa42 namespace: kserve-ci-e2e-test resourceVersion: "18250" uid: 988867c6-5cf4-4b53-a129-99db7c591829 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d996be08f4d namespace: kserve-ci-e2e-test resourceVersion: "18252" uid: 8b5fec85-d0f8-4a39-96ec-fcea8769f501 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d996c531985 namespace: kserve-ci-e2e-test resourceVersion: "18253" uid: 737d2530-985b-432c-bb9c-6366a5e7b699 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:24:13Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-01T18:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:24:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3d9e65a3f403 namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a541ce2e-1366-458a-b12f-b999b3799b23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:35Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m22.945s (2m22.945s including waiting). Image size: 8939170586 bytes.' metadata: creationTimestamp: "2026-07-01T18:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:35Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfadda1fb2 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: c8100b0f-38bc-4365-b781-eb4b372529f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:36Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfb3e6ec94 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: ca078ad2-03ec-45e7-8e0f-207033a36f14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:36Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfb532edf9 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: ccffc1af-5d6f-449a-8ce3-e31b04401542 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:36Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfb53eb5e9 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: af62d1e0-909c-4006-b93e-eac409369267 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:36Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfba8d60c2 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 5e4da34b-e277-44b8-b51c-8af82136a364 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:36Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfbb7d9771 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: e3aa2d7c-aa5e-4da1-a694-7fb81789ee32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:26:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:26:42Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dbfd96e1ddc namespace: kserve-ci-e2e-test resourceVersion: "19455" uid: 8004c741-583a-4f81-b191-7a38b13a8823 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:27:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dc595e51540 namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 7cd61828-4c99-421f-afa5-91500662b8fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 392cb330-b4f1-4d99-bd5e-9a2d6e172523 kind: Event lastTimestamp: "2026-07-01T18:27:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr.18be3dc595e5950f namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: e6df7e77-24fe-40e2-ae30-a29bef739a28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "18186" uid: f37d95d1-157b-4903-ab70-b25c9f0b355a kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-nfvrr' metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18be3d994ce510b5 namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: 87c08821-d00b-42e3-b6ae-c273bcd3fcab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18184" uid: 8ec3acb7-ab91-4876-8c65-caba72c5bd5d kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1-predictor.18be3d994835ee59 namespace: kserve-ci-e2e-test resourceVersion: "18188" uid: eb2eff18-45b5-4634-90d8-921080956d1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18185" uid: c2fbe78b-2c63-4afa-b483-d3e108d399c8 kind: Event lastTimestamp: "2026-07-01T18:26:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:26:51Z" name: isvc-autogluon-v1-predictor.18be3d9cc77ccdb0 namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: 489bde86-f99d-4054-abe4-d8fe26d58ae4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18185" uid: c2fbe78b-2c63-4afa-b483-d3e108d399c8 kind: Event lastTimestamp: "2026-07-01T18:26:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:26:51Z" name: isvc-autogluon-v1-predictor.18be3d9cc7f721c7 namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: d838fc0a-f1d6-4ee8-b02f-e8b0745361cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18175" uid: 2a265a66-f310-4fc7-92dd-4a172006d67e kind: Event lastTimestamp: "2026-07-01T18:26:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:26:52Z" name: isvc-autogluon-v1.18be3d9948385f2b namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: 60955b0d-d864-4b14-a3bd-9170a63e6681 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18175" uid: 2a265a66-f310-4fc7-92dd-4a172006d67e kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1.18be3d9948b130c0 namespace: kserve-ci-e2e-test resourceVersion: "18190" uid: cfbc5b93-e077-4e7d-a169-a34e3b1f2442 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18175" uid: 2a265a66-f310-4fc7-92dd-4a172006d67e kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:51Z" name: isvc-autogluon-v1.18be3d9948b1888b namespace: kserve-ci-e2e-test resourceVersion: "18191" uid: 282931af-72ae-4f48-a307-cde1836dfca9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:26:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: 2a265a66-f310-4fc7-92dd-4a172006d67e kind: Event lastTimestamp: "2026-07-01T18:26:52Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-01T18:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:26:52Z" name: isvc-autogluon-v1.18be3dc399998dee namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 9147028d-7b37-4383-8856-0b2e8d71723d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:27.274065Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03821d4e1f namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: c7594c4b-7044-497e-8d03-48d0bbcbd1c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-noslash-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03897ea3cd namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 1be4e9d3-6366-4eb3-b9db-4461a0a456e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:28Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:28Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03bf78f6cb namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: a66cec4c-5e4a-48c6-bbf6-7a2613bff4fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:28Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03c0d7c6c2 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: 077ab562-11ac-4b26-b051-cc8b08285c28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:28Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03c3fa1717 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: b562df4e-5170-473e-996a-4f77cd2584c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:28Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e03c468b9e1 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 765da87d-bd52-4d05-a348-214baf51ad3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:33Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e050e6214c7 namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: f1cd555a-7531-496d-98b6-a5799bc51c30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e05115439dd namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: e8daecaa-1b63-47eb-98ec-ff0050273e24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e0511c397e4 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 1ff2a7bd-2bd5-4a78-881e-b85c0badd6c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e0511cdbcaa namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 00635e50-dec6-43a7-bab0-89172c412e02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:34Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e0514a97c1a namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 9cc5a9c7-0b09-4199-8155-77b51ecf2ad7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:31:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:34Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e0515221b58 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: b74bcdec-afc6-4378-8507-091e458d7d00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:33:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e1d26f7a1b8 namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: c08a755a-7c46-4ab3-a70f-d416d3bbccf1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: ba09ef89-9e79-42d7-8362-73504bdde689 kind: Event lastTimestamp: "2026-07-01T18:33:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br.18be3e1d26f85125 namespace: kserve-ci-e2e-test resourceVersion: "22475" uid: ab92b67a-4c89-4db1-8546-1b6e283d9ec6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 4ab2717d-996f-40b1-8c45-a8047590024f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tw7br' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18be3e0381bfe427 namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: 64dc5d77-2467-46ad-88ec-42adc49c5389 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 597f55ad-65c8-4337-9dec-04f2a151f1be kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash-predictor.18be3e037c647beb namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: e664f52e-fc74-47e5-b595-65a323a7dc61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 28962310-0b9a-49d1-bfe9-0f57869feac8 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: isvc-autogluon-v2-noslash-predictor.18be3e06fc0f23b0 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: f46d3a10-c1eb-4ee9-952b-969a1497839f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 28962310-0b9a-49d1-bfe9-0f57869feac8 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: isvc-autogluon-v2-noslash-predictor.18be3e06fc824b92 namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 6845c71a-c14b-4549-a559-218fe8d099da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 28962310-0b9a-49d1-bfe9-0f57869feac8 kind: Event lastTimestamp: "2026-07-01T18:33:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:12Z" name: isvc-autogluon-v2-noslash-predictor.18be3e0a7b5bd1ca namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: 673e1d80-6448-4d8e-b1a1-6b7692585092 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 28962310-0b9a-49d1-bfe9-0f57869feac8 kind: Event lastTimestamp: "2026-07-01T18:33:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:12Z" name: isvc-autogluon-v2-noslash-predictor.18be3e0a7bca9a96 namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: b2e4c750-59fa-4be1-98f6-81371f921c08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 2dfccd1f-ccf3-4e83-b024-af0d0307d2bd kind: Event lastTimestamp: "2026-07-01T18:33:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:10Z" name: isvc-autogluon-v2-noslash.18be3e037c67bd50 namespace: kserve-ci-e2e-test resourceVersion: "22411" uid: 48462f71-fc75-4b4c-8ff9-4a29d4b06b93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 2dfccd1f-ccf3-4e83-b024-af0d0307d2bd kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash.18be3e037d0adf3f namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 51510fe4-7b7a-457f-8e17-152580401315 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 2dfccd1f-ccf3-4e83-b024-af0d0307d2bd kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:27Z" name: isvc-autogluon-v2-noslash.18be3e037d0b2933 namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 9e0e8a95-e290-499b-b613-77106ab92899 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: 2dfccd1f-ccf3-4e83-b024-af0d0307d2bd kind: Event lastTimestamp: "2026-07-01T18:33:10Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-01T18:33:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:10Z" name: isvc-autogluon-v2-noslash.18be3e1b9cc1901c namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: 84d33d5f-3e79-4937-89a9-f772979d1a11 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:27:06.617407Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-shssg to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:27:06Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dc6d1c44ea5 namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: b9a10509-e39a-47d6-afcc-e325bca500e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:07Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:27:07Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dc6ee33b161 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: d20001c4-a078-46f3-9580-fcb9cbff02ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:07Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dc6ef901508 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: e857d9d4-d21f-4d7d-b8d0-1b196a9ab3ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:07Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dc6f421b6b6 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: df4cc77d-7f62-408e-89cb-acfa3d7d7c44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:07Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dc6f51b1758 namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 27c9e472-69d3-4c08-b302-f9f2cb7ba0d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:27Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:27Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbc6933e00 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: 67189540-c945-4de3-a1f5-b4e7f9c6d3dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:27Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbca968b8c namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 1e772886-21c8-4381-8e83-43408c804980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:27Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbcafeabe3 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: b3e40594-f2d3-4d3c-80f0-d645a77f0972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:27Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbcb0a02d7 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 678df91c-e36e-4805-abc9-0020586fa00e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:28Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbce5151f1 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 2760af01-36e6-46f2-8cae-af1437fe0551 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:27:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:28Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3dcbceb7b840 namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: ec975158-c080-4f99-8a6b-62b3fd9e82bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:29:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:06Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3de2cb579ed0 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 75cc52d2-9975-4cb7-a74a-6e7ac98fda68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-shssg namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 364a1a1d-258d-4be1-b115-ec85f0e0502e kind: Event lastTimestamp: "2026-07-01T18:29:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:06Z" name: isvc-autogluon-v2-predictor-67b849d456-shssg.18be3de2cb582068 namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 5cf1668a-0336-4db9-aed9-0d96b66a7da5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 1e9af9dc-358b-4140-b152-b075c2ab3f10 kind: Event lastTimestamp: "2026-07-01T18:27:06Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-shssg' metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:06Z" name: isvc-autogluon-v2-predictor-67b849d456.18be3dc6d15d3def namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 211a0cd2-a218-44b0-bf58-43fadc8fc941 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 2dfca39d-e1de-4e8a-a2ee-de9324ecb318 kind: Event lastTimestamp: "2026-07-01T18:27:06Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:06Z" name: isvc-autogluon-v2-predictor.18be3dc6ccff7628 namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 6b02561a-f723-40ba-a5a4-83a4a1cb5b67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 1a71d9e5-4014-4671-b72a-23a9e871a8ac kind: Event lastTimestamp: "2026-07-01T18:27:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:21Z" name: isvc-autogluon-v2-predictor.18be3dca4b771249 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 0db4d95a-ef11-4883-a201-ec45cf95fc07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 1a71d9e5-4014-4671-b72a-23a9e871a8ac kind: Event lastTimestamp: "2026-07-01T18:27:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:21Z" name: isvc-autogluon-v2-predictor.18be3dca4be325dd namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: 6cf3d3e8-ebf6-48f5-870c-dbb132afdced reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 1a71d9e5-4014-4671-b72a-23a9e871a8ac kind: Event lastTimestamp: "2026-07-01T18:29:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:29:06Z" name: isvc-autogluon-v2-predictor.18be3dcdcc2b5dee namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 901b8d6e-94e3-45e0-8551-67011351bc4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 1a71d9e5-4014-4671-b72a-23a9e871a8ac kind: Event lastTimestamp: "2026-07-01T18:29:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:29:06Z" name: isvc-autogluon-v2-predictor.18be3dcdcca28ede namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 8f8c1263-3115-44f1-a2ac-0bd21fc598b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:29:11.920114Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:29:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3de3fe63f147 namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 4540a83b-4370-46a3-8c78-93a28091cbaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:12Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:29:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3de418155391 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 2835a81f-9a55-4c2c-b23f-ec3b28c82714 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3de41964ceb6 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: cd7af78f-d071-4a0c-b4c9-b500addf7d41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3de41be89cc0 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 801a0ef3-d2ca-437f-bda9-0d27f2c25fe8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3de41cad9e23 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 16eaaf2d-c015-4931-b9c4-0b75bd3bcc3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb1afc239e namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 76a04335-fc88-4f7a-a940-e9fc4910c8fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb1e499e8a namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: bf9ed8ea-2ce1-4e59-80c1-b42061571a24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb1eb015dc namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 42c93429-e7f8-410e-9cf2-30c0b57067ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb1eba4936 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 9c13720e-a1ec-4e25-baca-8b6bb5f2ac8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb21500f4a namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: bbfa3d1d-c957-4c86-bf01-4598145543e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:29:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:29:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3deb21b4958f namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 05c08774-d0d9-41b7-b89a-1fd80d791189 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:31:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:22Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3e024e263bb5 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: b6a9fe77-f923-4a3a-b81f-f364d8e92431 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 9f5f19dd-bf66-44fc-8b2a-c7b93b8dd39b kind: Event lastTimestamp: "2026-07-01T18:31:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:22Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx.18be3e024e275967 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: 8a15eca2-e6ad-40d3-bb65-709817b7c573 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3d322258-78bc-4af1-9b54-b1ef92c1687e kind: Event lastTimestamp: "2026-07-01T18:29:11Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-6g6bx' metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:29:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18be3de3fdee3856 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 045fa71d-54f0-4637-8775-6d3f4d3e9939 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: e89e8a47-bb83-4330-afb2-4418a2d31ff3 kind: Event lastTimestamp: "2026-07-01T18:29:11Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:29:11Z" name: isvc-autogluon-v2-variants-predictor.18be3de3f950b13d namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: ac426aaa-dfb9-4fe6-b8b4-dbafd5e2c3b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 05325a26-1bb5-44c0-9f08-8497165e056d kind: Event lastTimestamp: "2026-07-01T18:30:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:11Z" name: isvc-autogluon-v2-variants-predictor.18be3de777e5b3e1 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 2a04f3fe-e2d8-4920-a481-be5259b45adf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 05325a26-1bb5-44c0-9f08-8497165e056d kind: Event lastTimestamp: "2026-07-01T18:30:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:11Z" name: isvc-autogluon-v2-variants-predictor.18be3de778523582 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 511e15fe-519b-43ae-a3b6-dc17e33be927 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 05325a26-1bb5-44c0-9f08-8497165e056d kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: isvc-autogluon-v2-variants-predictor.18be3df5754a51e7 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: c1ed8b92-56d0-468a-93c8-1886f7112404 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 05325a26-1bb5-44c0-9f08-8497165e056d kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: isvc-autogluon-v2-variants-predictor.18be3df575b37cd2 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: 04463864-3a97-42f6-a23e-2f71a0d05bec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:29:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: f2438a36-636e-4272-91b7-fc3a5e83974c kind: Event lastTimestamp: "2026-07-01T18:31:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:20Z" name: isvc-autogluon-v2-variants.18be3de3f94bfe91 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 38bb3d7e-619a-4231-81a6-ef655bddeb05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:29:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: f2438a36-636e-4272-91b7-fc3a5e83974c kind: Event lastTimestamp: "2026-07-01T18:29:11Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:29:11Z" name: isvc-autogluon-v2-variants.18be3de3f9f60fbd namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 965028fa-6204-464a-bc85-ac17767a63d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:29:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: f2438a36-636e-4272-91b7-fc3a5e83974c kind: Event lastTimestamp: "2026-07-01T18:29:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:29:11Z" name: isvc-autogluon-v2-variants.18be3de3f9f66af9 namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: e74fff26-f56d-4d5c-9ecb-4b68931d5ff3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: f2438a36-636e-4272-91b7-fc3a5e83974c kind: Event lastTimestamp: "2026-07-01T18:31:20Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-01T18:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:20Z" name: isvc-autogluon-v2-variants.18be3e01ffd3021e namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 53063089-d11d-4736-911a-4435acfc4c1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:27:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: c4e55d08-bf91-469d-be7b-0256bd39f274 kind: Event lastTimestamp: "2026-07-01T18:29:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:29:04Z" name: isvc-autogluon-v2.18be3dc6cd064583 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 3aa2e125-2c6c-4beb-a5bf-98805069b373 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:27:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: c4e55d08-bf91-469d-be7b-0256bd39f274 kind: Event lastTimestamp: "2026-07-01T18:27:06Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:27:06Z" name: isvc-autogluon-v2.18be3dc6cd7e7a60 namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: 7550e346-b8db-433a-9a34-4e95a6cdecc8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:27:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: c4e55d08-bf91-469d-be7b-0256bd39f274 kind: Event lastTimestamp: "2026-07-01T18:27:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:27:06Z" name: isvc-autogluon-v2.18be3dc6cd7ecb53 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 0af6aaa4-afdc-491f-a1a5-3cded5eec174 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:29:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: c4e55d08-bf91-469d-be7b-0256bd39f274 kind: Event lastTimestamp: "2026-07-01T18:29:04Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-01T18:29:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:29:04Z" name: isvc-autogluon-v2.18be3de255b2d1b0 namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: a7c49434-a841-4ba3-bc32-48303849825d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:22:15.827649Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17425" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-7997fc58c6-pbljd to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:22:15Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d831d58b869 namespace: kserve-ci-e2e-test resourceVersion: "17429" uid: 72f240a5-3c7e-403f-8840-dd755251e679 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:16Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d832e61406e namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: 1324907a-a1da-43ce-a36a-f13315b937d1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:16Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:22:16Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d835ad8a73a namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: 7af03b7b-69f4-48f0-b8b2-dac94ccc02c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:16Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d835c3d466e namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: d9e0b5c6-f2cd-4cde-ad6a-5f9594ec1f47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:16Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d835ec3c43b namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: 88cbd34b-9359-44d1-ab14-6d54989d5d69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:16Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d835f28aaed namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: e790b7d2-9640-4914-9450-bce598f4ff44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d846f9f7472 namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: c4494238-785f-4d4a-857a-e506d13ea19f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d847252f9c8 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: b1e14214-3af2-459c-89e0-2048e9b2ef2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d8472e0a7c1 namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: 9d959d87-bd53-426c-a962-d5c9f627ad1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d8472eb7ea7 namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 207c55b7-2272-4881-9324-0a63aa280eca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d84760c6773 namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: 3becb0a5-61a0-4928-ac19-4e84df8396ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d8476721ab0 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: adf0cda6-54e3-4d42-9ea9-d914231c8784 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d84767ccbf8 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: fdd20a88-426f-4ba0-9391-cf77914d09f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d84794d3b87 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: c21f5fac-e607-4606-a35d-9ab8be7c8f75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:22:21Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T18:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:21Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d8479bd4b19 namespace: kserve-ci-e2e-test resourceVersion: "17509" uid: 6bd27cc0-28dc-4484-8df4-e06bc47c4c69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:23:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:58Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d84abaa6bfc namespace: kserve-ci-e2e-test resourceVersion: "18311" uid: f533162b-5ef8-4a6e-aa1c-cacfe12b2295 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:22:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:23:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:58Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d84e7a21058 namespace: kserve-ci-e2e-test resourceVersion: "18312" uid: 7e009a7e-07a6-4c42-a8d4-63305ad7a8c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d994cdea184 namespace: kserve-ci-e2e-test resourceVersion: "18210" uid: 3fac579f-91f9-4554-878a-ebece0f68dee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d994cded640 namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: 3543eec7-efc5-48de-be5d-388d64c7c921 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d994cdfb702 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: 381f8600-9d9e-480f-b91b-ee571e8e3935 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:23:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-pbljd namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 95c3687d-82e0-47aa-9ec2-ec48ac026a8c kind: Event lastTimestamp: "2026-07-01T18:24:04Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:24:04Z" name: isvc-logger-predictor-7997fc58c6-pbljd.18be3d9a17d896b9 namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: b0ab9f31-5375-4889-ab8a-316015fc8f72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-7997fc58c6 namespace: kserve-ci-e2e-test resourceVersion: "17414" uid: f5e8a311-5e11-40a3-9457-8c28ff376747 kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: 'Created pod: isvc-logger-predictor-7997fc58c6-pbljd' metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:15Z" name: isvc-logger-predictor-7997fc58c6.18be3d831d03241c namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 5a0f28b6-0eb1-42e5-9edb-d180159c2d4e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: 4be89d81-442b-4052-845b-9fa10acec990 kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: Scaled up replica set isvc-logger-predictor-7997fc58c6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:15Z" name: isvc-logger-predictor.18be3d83191083e3 namespace: kserve-ci-e2e-test resourceVersion: "17416" uid: 7aa748eb-a41d-4c5a-bdc9-4804b915dce0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 24562aaf-8e61-449a-9312-c788af2f24d3 kind: Event lastTimestamp: "2026-07-01T18:22:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:45Z" name: isvc-logger-predictor.18be3d8698417687 namespace: kserve-ci-e2e-test resourceVersion: "17696" uid: d3362260-0e44-4900-8e54-a5cadbebc511 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 24562aaf-8e61-449a-9312-c788af2f24d3 kind: Event lastTimestamp: "2026-07-01T18:22:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:45Z" name: isvc-logger-predictor.18be3d8698c0bcf7 namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: f5f2482e-3dab-4330-84eb-f96be276f164 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:23:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 24562aaf-8e61-449a-9312-c788af2f24d3 kind: Event lastTimestamp: "2026-07-01T18:23:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:45Z" name: isvc-logger-predictor.18be3d8d96d72b84 namespace: kserve-ci-e2e-test resourceVersion: "18133" uid: 90bd46db-f4d9-4d88-9240-388188acd36f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:23:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 24562aaf-8e61-449a-9312-c788af2f24d3 kind: Event lastTimestamp: "2026-07-01T18:23:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:45Z" name: isvc-logger-predictor.18be3d8d97467d6e namespace: kserve-ci-e2e-test resourceVersion: "18134" uid: 895c6e40-24e4-470b-bcf2-daf8c7915ced reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: 75ba28bd-e723-45b8-975b-7c44cb8bee28 kind: Event lastTimestamp: "2026-07-01T18:23:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:38Z" name: isvc-logger.18be3d831902d5bc namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: 113b9ef1-225c-4e3a-afe2-66814b420aea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: 75ba28bd-e723-45b8-975b-7c44cb8bee28 kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:15Z" name: isvc-logger.18be3d83195ee789 namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: 26ccc680-1eb3-4e21-bbe3-6128b264d7e0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: 75ba28bd-e723-45b8-975b-7c44cb8bee28 kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:15Z" name: isvc-logger.18be3d831fbfd45d namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: 5e28d311-2e9e-4348-ac52-3625008c18df reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "18081" uid: 75ba28bd-e723-45b8-975b-7c44cb8bee28 kind: Event lastTimestamp: "2026-07-01T18:23:38Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-01T18:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:38Z" name: isvc-logger.18be3d966ec9e733 namespace: kserve-ci-e2e-test resourceVersion: "18082" uid: 0ef3a2c0-f39a-401d-9ac3-c00a0e8d0f25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:20:30.626354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6a9edc5154 namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 3365dc1a-e561-4502-a711-64de5430867e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:31Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:20:31Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ab876a23d namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 9a3a79cf-1885-46af-93a8-9afac03d8afb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:31Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ab9dd238c namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 9c9ed489-1460-4553-ad57-deaa3ffd486a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:31Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6abd0d6bfa namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 8e141c31-0c24-415c-9411-c38e49ce997c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:31Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6abd870432 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: 0e73ae6f-f55b-46b4-9779-83b82728b702 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ba5f0b1ff namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: bb844b9e-6696-45f2-875e-f60b8e126873 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ba921d0b8 namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 9a1ccf05-b02e-4c96-9dac-5d80e477c510 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ba987df7a namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 1042cf2e-1fba-4509-943c-d4b4d5187e3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6ba9934af2 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 299e8d33-8883-44f2-9dc8-b7437f9c062f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6bac449171 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: eb45745d-f493-48be-a4d5-7273ccd23c1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6baca95c23 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 7e1d1761-413d-4a7e-90bb-fd644c113ba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6bacb2f8ca namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 5ee5c2ec-2189-4566-9ae8-385da1194322 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6bafff32d7 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 5d658f5c-ad34-4cb6-ba33-a50532e705e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:20:35Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T18:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:35Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6bb060bc83 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 7920a211-2a08-49e2-9275-ea51f292a602 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:17Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6c1deef6f5 namespace: kserve-ci-e2e-test resourceVersion: "17467" uid: 093b9c3a-a3cc-4a1d-9dbb-570327eda877 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d6c1df8a9ed namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 4ceb8756-9ecb-4bb8-8713-7dc3a37807ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:05Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d80c728aed0 namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: d0ef0ece-eef5-4cef-9f2d-a36c8ddc0ee6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:05Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d80c729afb4 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: c640f956-144a-431c-866d-cb65f5c75fc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:05Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d80c72a179f namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: df91279d-b680-4a40-b240-b27383d8ecf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 63550692-e449-4ff4-9c47-a03bdd856474 kind: Event lastTimestamp: "2026-07-01T18:22:13Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:13Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4.18be3d814e280d5a namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: f76b0edb-4659-430f-a9d6-6d094c0dd894 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 8f7ce29d-e146-4e41-9295-6476e282800f kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-pvdk4' metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb.18be3d6a9e8a862f namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 355669c1-ee1c-417f-8862-e26b14621fa6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 817bf457-dbef-4480-8d6e-787efec32e2b kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-7fd7b677bb from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-custom-predictor.18be3d6a99383770 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 8e4743b2-37bf-4159-a5ed-3abd9af2ad56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 2cbaff3c-508c-4cd7-a071-9416933eea95 kind: Event lastTimestamp: "2026-07-01T18:21:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:00Z" name: isvc-sklearn-batcher-custom-predictor.18be3d6e179ff80a namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: c459dd21-a042-40f5-aa26-a8dd1dadc8c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 2cbaff3c-508c-4cd7-a071-9416933eea95 kind: Event lastTimestamp: "2026-07-01T18:21:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:00Z" name: isvc-sklearn-batcher-custom-predictor.18be3d6e182ac6a7 namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: 0e4d7e67-e8d8-4845-8bbf-c8552185952a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 2cbaff3c-508c-4cd7-a071-9416933eea95 kind: Event lastTimestamp: "2026-07-01T18:22:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:00Z" name: isvc-sklearn-batcher-custom-predictor.18be3d75167571b0 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: dee225e0-8387-43b1-ae8d-f940ab2b7671 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 2cbaff3c-508c-4cd7-a071-9416933eea95 kind: Event lastTimestamp: "2026-07-01T18:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:00Z" name: isvc-sklearn-batcher-custom-predictor.18be3d7516ffc873 namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 5cc75cff-8461-4ed0-bb5b-1bd91f9045d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: b9323c08-43d3-4726-a733-0ce64f72eeb1 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-batcher-custom.18be3d6a990dd2e7 namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: 051162ca-0e02-4dcb-8f20-f6d75b1590b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: b9323c08-43d3-4726-a733-0ce64f72eeb1 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-custom.18be3d6a9966733b namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: a36a3953-e434-4993-b454-9d23c62e84b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: b9323c08-43d3-4726-a733-0ce64f72eeb1 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-batcher-custom.18be3d7eec006e27 namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: da74df8e-22d8-4cfc-8689-95eee0f4e07f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:18:45.340264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d521b521dc0 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: cac5a6e8-e984-4950-9020-f43cbc5f9df0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d523665a4a3 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5d87f1aa-47fd-4559-be6a-aff8b3df4451 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d5237da81e0 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 45fe3be7-b637-448e-89d7-f15447420f5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d523aa82d9a namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 85a21eec-3d50-4a57-923e-1c50c8938250 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d523b130f59 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: bcd30327-9f09-4651-949e-3a83b7c75dec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:18:50Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:18:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:50Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d5354393613 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 6d4bf485-4efc-40dc-9279-a4699747167b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 15.285s (15.285s including waiting). Image size: 1559186858 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d56e34f51bf namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: c9033cc9-ea6f-450c-89bd-68766175fce9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d56e664595d namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 1f602f51-a05b-4342-bf23-437d08a2d3e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d56e6e18b2b namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 62b51460-1fff-4595-950e-ffba653bca70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d56e6ef97e4 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 438a1caf-d466-40ab-9f7f-bd0c0f56a65c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:08Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.538s (2.538s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:08Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d577e439f11 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 4681c923-9ebb-49ed-851e-535e0d62f27b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:08Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d5781057806 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: ea32cf7d-1095-4fcd-b42b-e9379123ffa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:08Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d57817ef30e namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 627fcf61-2fd9-4104-a914-ecd78d13a6dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:08Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" metadata: creationTimestamp: "2026-07-01T18:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:08Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d57818c0259 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 8286a2cc-fb88-4e4e-a006-03f6391dd07c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" in 2.669s (2.669s including waiting). Image size: 239994467 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d5820aafe96 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 4bfe4e5c-d74e-4600-a859-f13ac8430ed7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:11Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T18:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d58242f6ef3 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a537a635-f099-4dce-afee-458fa34c7849 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:19:11Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T18:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d5824a2f9d4 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 43970095-0fb4-4ade-bcae-2825f9e506de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:19:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:44Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d587a5ea406 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: d7e24667-b51c-4c1c-810f-11e654cbfc91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:19:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:44Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d587a67aa9c namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: ba5ae6cc-44ca-43f1-a9f9-90a111505d6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d6aa0f6d5ec namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: daa98384-e632-4c6a-b28d-8b5ad20ad182 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d6aa0f7683d namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 5f822679-44f8-4f0a-b299-1629cf3f7cde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d6aa0f9113c namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 48a9ae43-de58-4e20-856f-9e402b5dfda7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8977a0ba-61f3-4a73-9364-315bb2cc6a65 kind: Event lastTimestamp: "2026-07-01T18:20:43Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:43Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb.18be3d6b568a63cf namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: bfde251c-8918-4175-86af-87a588f6feb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-54dcb69b4c namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 6ed8fced-f0b7-41ec-8285-1ddf20165c92 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: 'Created pod: isvc-sklearn-batcher-predictor-54dcb69b4c-46mbb' metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c.18be3d521aeafe1f namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 7eb3ab1a-10d9-4052-a285-fe260cbd6d76 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 0abb4855-0669-4ea8-8de8-12cb1994ca0b kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-54dcb69b4c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher-predictor.18be3d52164b04c0 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: f8c202d3-6af2-4803-8578-e2133b62578e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 8d2dd14e-6d3b-49f1-b90f-e5e797f70797 kind: Event lastTimestamp: "2026-07-01T18:19:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:45Z" name: isvc-sklearn-batcher-predictor.18be3d559bf49fb1 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: 5f28b38a-40c5-4299-b5c3-37c21cc2a2b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 8d2dd14e-6d3b-49f1-b90f-e5e797f70797 kind: Event lastTimestamp: "2026-07-01T18:19:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:45Z" name: isvc-sklearn-batcher-predictor.18be3d559c494761 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 7d181141-0400-43c5-85be-a7384e52a12d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 8d2dd14e-6d3b-49f1-b90f-e5e797f70797 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-predictor.18be3d639a4127d0 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 7296b758-e9b1-42c3-b1c7-0378bba55f4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 8d2dd14e-6d3b-49f1-b90f-e5e797f70797 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:30Z" name: isvc-sklearn-batcher-predictor.18be3d639ab0bfbf namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 927436c5-7590-4947-88d0-060a020d57b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: ccf22d6d-c764-43c3-997d-68036ded1b6e kind: Event lastTimestamp: "2026-07-01T18:20:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:24Z" name: isvc-sklearn-batcher.18be3d521c5d8b1d namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: 47d589f9-09ef-40d1-93e4-bdb425034967 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: ccf22d6d-c764-43c3-997d-68036ded1b6e kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:45Z" name: isvc-sklearn-batcher.18be3d521cabcd73 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 0d90ed0a-fc53-4844-8d9e-cae1e012fa76 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: ccf22d6d-c764-43c3-997d-68036ded1b6e kind: Event lastTimestamp: "2026-07-01T18:20:24Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-01T18:20:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:24Z" name: isvc-sklearn-batcher.18be3d69440284e5 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 83316ea9-af73-48c2-86cd-2299e57b7a1c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:22:05.768591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-4kl4l to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:22:05Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d80c5c7b025 namespace: kserve-ci-e2e-test resourceVersion: "17282" uid: 8488f5dd-c13c-4589-b7de-25aa59860be4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf5755ff8-q2qll source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:06Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:06Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:22:06Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d80df3e8c03 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: c2d70724-4896-4e3c-bd24-9c56983e793c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:06Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-01T18:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:06Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d80e07ecf3b namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 71e0f33c-70ea-4654-a726-48531efba107 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.228s (1.228s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d8129b3d0c9 namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: e1aa68ea-8851-4352-a5fe-27d701f6f80e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d812dcbbf7c namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: 3450c445-10a6-428a-a44c-cddc026e74ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d812e3967d0 namespace: kserve-ci-e2e-test resourceVersion: "17325" uid: edff5e9c-062c-4e69-94a1-8756f337567c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d812e444aca namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: d140f001-cb3c-495b-a3ab-7f98901e486f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d81313b5674 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: b39fb0fe-6c09-4836-a700-532ee4f397ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:07Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d8131a94df2 namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: 03d4c9f9-2d18-45b5-99a9-0760ca45de31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d9950245b17 namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: f040f98a-c274-4403-a4d9-83a33e25deaa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-4kl4l namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: dd590e93-550f-41b3-8c34-66f61639da69 kind: Event lastTimestamp: "2026-07-01T18:23:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:51Z" name: message-dumper-predictor-6575bcfdfd-4kl4l.18be3d995024c43d namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: 7f9ab230-8aaa-4cc9-951b-818fb0d84353 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 194d9005-005b-4f3f-a729-51b7f69fe367 kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-4kl4l' metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:05Z" name: message-dumper-predictor-6575bcfdfd.18be3d80c5b9b1c4 namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: dc42a564-48c5-4fa3-9a33-1cf27945883d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: db864fd6-cdc1-4aa9-ae5e-260c5cdd977a kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:05Z" name: message-dumper-predictor.18be3d80c20785a9 namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: 8ff31587-3053-4e8a-9a3f-ae5d34cf597b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: 7db13505-1dbb-4eb4-909a-d6eafec6a4a7 kind: Event lastTimestamp: "2026-07-01T18:22:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:50Z" name: message-dumper-predictor.18be3d8440bbe9e9 namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: 9084cba6-355a-48be-8c27-d504c0209fd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: 7db13505-1dbb-4eb4-909a-d6eafec6a4a7 kind: Event lastTimestamp: "2026-07-01T18:22:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:50Z" name: message-dumper-predictor.18be3d84412bcbaf namespace: kserve-ci-e2e-test resourceVersion: "17733" uid: 0525368e-3e44-4434-b553-6fbb5fe4ef5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 7db13505-1dbb-4eb4-909a-d6eafec6a4a7 kind: Event lastTimestamp: "2026-07-01T18:22:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:35Z" name: message-dumper-predictor.18be3d87bff75ee4 namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: f3791f3e-2aa9-465e-ba51-5390f7c748ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 7db13505-1dbb-4eb4-909a-d6eafec6a4a7 kind: Event lastTimestamp: "2026-07-01T18:22:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:35Z" name: message-dumper-predictor.18be3d87c06612fa namespace: kserve-ci-e2e-test resourceVersion: "17626" uid: 4855b9c8-81e5-47bd-8a28-550c14eeb2b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: 0dd529a2-c4b7-4788-acff-61f465b2d47d kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:15Z" name: message-dumper.18be3d80c219c396 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 5965e8eb-a7b5-4d00-8d6c-b2b7d2a1562c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: 0dd529a2-c4b7-4788-acff-61f465b2d47d kind: Event lastTimestamp: "2026-07-01T18:22:05Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:05Z" name: message-dumper.18be3d80c290c213 namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 47047bdf-596b-4138-8bf6-4894cbbe2276 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17397" uid: 0dd529a2-c4b7-4788-acff-61f465b2d47d kind: Event lastTimestamp: "2026-07-01T18:22:15Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-01T18:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:15Z" name: message-dumper.18be3d830beb303d namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: e03a7cba-5911-42d3-a01d-51bdc3a0abb7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "23054"