--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:44:10.482411Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c58593aa2 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: 09918faa-f59b-4732-95d4-e2b6cb236bfe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-ts-v1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c621fb8ca namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 79646f32-1726-4997-a9e4-d3db1872c94a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:11Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:44:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c95ebf055 namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: 4d3b3f09-e6c4-4295-8080-a07f0c848b74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c97692b5d namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: cb0e6314-7359-4c2e-bf7d-a72e40c307f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c9a2d088f namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: cbdccc4e-f199-46c2-8dd7-20eb1813cb42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971c9a9d9aac namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 769c38ea-8245-430c-a156-007dc1d450ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:19Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T21:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:19Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e8d61b595 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 3963d024-a077-4abc-9109-6d77fdf818ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e90470f82 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 620bf57c-0708-479a-abfe-0997cabd22f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e90c8094d namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: d0495f6a-5a0d-45b3-9596-7a5b4a8c1704 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e90d41b4c namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: c5bd0d19-b5a6-45e7-beab-3bee68d6f542 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e93bbaaca namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: e689d6fd-ad84-42a1-81b9-1c2f9359b9a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:20Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971e94308eda namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: cbb66321-678b-442d-b417-de4af67e9868 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:27Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be971f053e8b2a namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 66bfc63c-aac7-451a-92f7-03018abbf50a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:40Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be972357e80dfb namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: db3f9dd1-8d73-4190-b412-5f6af9879a8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: ca50158d-ce0c-4ebd-bddc-c87df7753b08 kind: Event lastTimestamp: "2026-07-02T21:44:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:40Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv.18be972357ea8231 namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: 34a440fa-44cc-4ea7-b3f2-12997a6c554f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: af4b9278-8f33-4988-9e79-7f0d22af8619 kind: Event lastTimestamp: "2026-07-02T21:44:10Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-rfhkv' metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18be971c57bd85a7 namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: add90533-cf7e-4b6d-8bc6-89d03749cafc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: d3d21d1b-5038-41c0-bd0e-9034eb1b271c kind: Event lastTimestamp: "2026-07-02T21:44:10Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1-predictor.18be971c532b833d namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: d0a8d1d5-1462-483d-bc1b-bcd246be410c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 0e291251-391d-4c2c-a904-20db592f116b kind: Event lastTimestamp: "2026-07-02T21:44:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:40Z" name: isvc-autogluon-ts-v1-predictor.18be971fd20a90c6 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 031a09e8-492e-4703-bf09-a08629f611e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 0e291251-391d-4c2c-a904-20db592f116b kind: Event lastTimestamp: "2026-07-02T21:44:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:40Z" name: isvc-autogluon-ts-v1-predictor.18be971fd27336e8 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 866e590a-20b0-4384-b266-4542d4c87d32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: d7f34c04-d9ae-440d-bb8f-06bc0fbdfcba kind: Event lastTimestamp: "2026-07-02T21:44:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:38Z" name: isvc-autogluon-ts-v1.18be971c5335a66e namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 5ae204fa-aaa7-4d16-9e7a-6bb356a2c390 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: d7f34c04-d9ae-440d-bb8f-06bc0fbdfcba kind: Event lastTimestamp: "2026-07-02T21:44:10Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1.18be971c53a39a3e namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: 5014423d-aae4-4977-95c8-b4e060f4d335 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: d7f34c04-d9ae-440d-bb8f-06bc0fbdfcba kind: Event lastTimestamp: "2026-07-02T21:44:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:10Z" name: isvc-autogluon-ts-v1.18be971c53a3fab4 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 6173781e-3706-42b3-899e-27dbcb81747e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: d7f34c04-d9ae-440d-bb8f-06bc0fbdfcba kind: Event lastTimestamp: "2026-07-02T21:44:38Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-02T21:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:38Z" name: isvc-autogluon-ts-v1.18be9722c576232b namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 9f15f682-ea3a-411f-aa6e-79a23ac0da68 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:34:29.095963Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be9694fb041471 namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: fc555b17-460a-4a61-a8c3-8fe6535aca0e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be969515150ac5 namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: 2dd34bf1-7da0-414f-94f2-ddec79b35c29 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be9695168835cc namespace: kserve-ci-e2e-test resourceVersion: "17543" uid: 2b082a5c-e756-4a64-b9e7-703248aef247 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be969519557733 namespace: kserve-ci-e2e-test resourceVersion: "17544" uid: 4a35970d-96e6-4dfd-bf87-2cb43cf1b1db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be969519cd4828 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 3fa09de4-7a5e-46c7-a224-c5455422e315 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:34:59Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-02T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:59Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be969c056f89e4 namespace: kserve-ci-e2e-test resourceVersion: "17767" uid: 8c124266-397e-4c34-91d9-6b0a7d89d106 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:11Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m12.441s (2m12.441s including waiting). Image size: 8939170586 bytes.' metadata: creationTimestamp: "2026-07-02T21:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:11Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96badb8bdc75 namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: a2befed4-da6e-4285-b75a-535e2617a428 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96bae9db19a6 namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: 48ed8efb-b9bf-4be1-a4a6-ff3f1bc314dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96baea99e49d namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: cec06660-2b6c-472c-ac3c-901595e8aab0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96baeaa6a6d4 namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2fa07c97-7f15-4dc9-8949-b1497976bf77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96baf1f46401 namespace: kserve-ci-e2e-test resourceVersion: "18691" uid: babdab02-dee5-44ba-aa02-2c436637a427 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96baf31a79a0 namespace: kserve-ci-e2e-test resourceVersion: "18692" uid: 57287928-fb56-4d99-b364-442a0fcaf9e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96bb6033455d namespace: kserve-ci-e2e-test resourceVersion: "18918" uid: e599d980-d617-4ead-9aee-3afbc6caad7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96c13fb0f080 namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2fc5d26d-cef6-4933-a20b-ebc21119b0c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96c13fb17e4d namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: fdef5b29-0fc1-4c2f-9eff-0ae9d43a5da2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ff252377-bb33-498f-b199-fbc1221ea67e kind: Event lastTimestamp: "2026-07-02T21:37:40Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp.18be96c16e384009 namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: 0dfb59e1-0894-4d6f-a619-2498609e5585 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17480" uid: df632580-6060-4124-8d89-b67a7eb38841 kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-9rzcp' metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18be9694fa5f8725 namespace: kserve-ci-e2e-test resourceVersion: "17510" uid: 7a72dbf2-8a73-4d91-ba3f-fae01fc3730f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: 120d908d-eca5-4ac2-87c0-9b9e454dd8b7 kind: Event lastTimestamp: "2026-07-02T21:34:28Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:28Z" name: isvc-autogluon-v1-predictor.18be9694f4b360ea namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: 59d675c2-e6b6-4fc1-8ca9-3d45116ae1a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:34:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 9a69b47b-c2bf-4aa4-969a-3a30577979e8 kind: Event lastTimestamp: "2026-07-02T21:37:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:37:29Z" name: isvc-autogluon-v1-predictor.18be96987366e428 namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: a81a375f-4ad9-494d-8ad0-f09ee7fdf6d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:34:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 9a69b47b-c2bf-4aa4-969a-3a30577979e8 kind: Event lastTimestamp: "2026-07-02T21:37:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:37:29Z" name: isvc-autogluon-v1-predictor.18be969873c8f139 namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 033cb3ed-b921-4891-9a15-fe6499ed8874 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:34:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: 5e0be1e9-b50e-437e-9a89-ccd5a4104b91 kind: Event lastTimestamp: "2026-07-02T21:37:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:37:30Z" name: isvc-autogluon-v1.18be9694f4b6ee24 namespace: kserve-ci-e2e-test resourceVersion: "18829" uid: d60ae264-53b5-47e8-96eb-e9b38f7f9bf0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: 5e0be1e9-b50e-437e-9a89-ccd5a4104b91 kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1.18be9694f568b27b namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: 3555cebf-6139-47c2-be0c-2383a3da175c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: 5e0be1e9-b50e-437e-9a89-ccd5a4104b91 kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:29Z" name: isvc-autogluon-v1.18be9694f56901b8 namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: 8a0db799-af9f-404e-b397-22655e2efd9e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18827" uid: 5e0be1e9-b50e-437e-9a89-ccd5a4104b91 kind: Event lastTimestamp: "2026-07-02T21:37:30Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-02T21:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:37:30Z" name: isvc-autogluon-v1.18be96bf1f87caee namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: d18d9e11-0a68-44fa-8a36-dfce381879be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:42:05.172533Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff2b4c4b8c namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 5dea1126-1182-4dea-8917-117277bf9c3e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-noslash-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff323a5ac1 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 344deea5-231b-4e76-9319-8237a62d2b9c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:06Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:42:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff68efe37c namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 36991d46-5884-4d67-b392-a2979f4bbc34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff6a4b836d namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 61e63e88-d0fe-4c82-8a62-623a83d063d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff6cecff61 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 53b7d835-8a7b-4ce3-81cf-4ce7c0ed1f80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be96ff6d5d40fd namespace: kserve-ci-e2e-test resourceVersion: "20972" uid: 38121056-a0bf-4081-acef-37d86b07af6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be97045abfb159 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 3695fb32-0116-484c-b437-80cffb841b51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be97045eafa85b namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 8d43d401-9db8-4b4b-b52f-202496bfe806 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be97045f33b25f namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 0372847f-b3fb-482e-81fc-78a7c05d24c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be97045f40e3b9 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 2e5bb201-4434-459e-9871-a3019a320847 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be970462284c38 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 771e7ad9-6de0-4240-b9c3-b55bffcaee52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be97046298a88b namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 199d33b3-3144-4032-bcab-67dc8d81bf37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:44:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be971b25da6b4b namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: 759fa785-4fba-41f4-aae3-e0a229906592 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:44:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be971b25db023e namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: c3662996-6ce2-4365-8ccf-5ca55d330781 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 0738e969-6412-41b7-99bb-1d6f152d0cbf kind: Event lastTimestamp: "2026-07-02T21:44:05Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc.18be971b2ce97cd9 namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: fbeaff9d-853c-44b9-a9cd-8d28bb3c2695 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 47cb3447-c425-49c1-9202-072cfb35ef47 kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-92nmc' metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18be96ff2ae8773e namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: c657345d-5c58-4db5-b09b-477f777c091d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 69b79f1f-8fdc-4940-91bc-a081dc873cec kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash-predictor.18be96ff25e6f38f namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 8c94f767-dac7-48fd-8dae-217945fa32f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 1e059e0d-a86f-4e5c-8fc2-7cb85db73a78 kind: Event lastTimestamp: "2026-07-02T21:42:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:20Z" name: isvc-autogluon-v2-noslash-predictor.18be9702a4e90a4d namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 6ea5ccf4-9f23-4092-b2a3-d03fda7e9b1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 1e059e0d-a86f-4e5c-8fc2-7cb85db73a78 kind: Event lastTimestamp: "2026-07-02T21:42:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:20Z" name: isvc-autogluon-v2-noslash-predictor.18be9702a58c5774 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 124715d2-5404-457b-841c-930a09042133 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 1e059e0d-a86f-4e5c-8fc2-7cb85db73a78 kind: Event lastTimestamp: "2026-07-02T21:44:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:05Z" name: isvc-autogluon-v2-noslash-predictor.18be97062333237d namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 8da2df01-8728-421f-ba1a-a1569675abb7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 1e059e0d-a86f-4e5c-8fc2-7cb85db73a78 kind: Event lastTimestamp: "2026-07-02T21:44:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:44:05Z" name: isvc-autogluon-v2-noslash-predictor.18be970623854178 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: 4f62ff77-e96e-45ff-893c-096b1e17b92c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 6b9a47a7-0637-4d6b-ac6d-9173fa3e5a3b kind: Event lastTimestamp: "2026-07-02T21:44:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:04Z" name: isvc-autogluon-v2-noslash.18be96ff25ef5085 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 2823886d-062a-41db-a160-74be7314cf9a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 6b9a47a7-0637-4d6b-ac6d-9173fa3e5a3b kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash.18be96ff2688c9a9 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: a3c2826b-3420-488b-a9fa-30329f0909d4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 6b9a47a7-0637-4d6b-ac6d-9173fa3e5a3b kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:05Z" name: isvc-autogluon-v2-noslash.18be96ff2689125a namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: db446235-661d-45ac-ac0e-8605ce32e3a0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: 6b9a47a7-0637-4d6b-ac6d-9173fa3e5a3b kind: Event lastTimestamp: "2026-07-02T21:44:04Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-02T21:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:44:04Z" name: isvc-autogluon-v2-noslash.18be971adf19012b namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: 5d7a4807-5a79-40d2-8f71-7497cdd4189c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:37:44.549158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-bmxbr to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:37:44Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c27cef2a3e namespace: kserve-ci-e2e-test resourceVersion: "18977" uid: f2a69712-0c4f-4179-9be7-bc2010fc87ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18985" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:37:45Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:37:45Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c29964c4d4 namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: 763ba08d-6ab5-4153-8519-c684c8a92c40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:37:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:45Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c29ac95c45 namespace: kserve-ci-e2e-test resourceVersion: "18996" uid: 498ae7ae-508c-4ce3-8e3a-db38e99dc0e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:37:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:45Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c2a01d3cf9 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: fd0e537a-5347-4bfa-8dd8-d97a1f6254ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:37:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:37:45Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c2a0d2fe92 namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 0743a01f-1641-428a-a086-a3a9451dcec3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c96926f824 namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 63109277-4aa4-40cc-b480-93c6b964976f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c96c9113aa namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 0cc8414a-0c1f-46ae-ae4a-4fbc9f51ebe2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c96d0ca9ed namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: e0a8ef92-8326-4923-aa5e-97dc51a83166 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c96d17a179 namespace: kserve-ci-e2e-test resourceVersion: "19209" uid: 7901e10b-aac5-4cf5-846d-9de5b98364f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c9704a4309 namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: 329651fb-3412-434a-a8b1-a190ab757f47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:38:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:38:14Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96c970c81b07 namespace: kserve-ci-e2e-test resourceVersion: "19212" uid: 0f728f14-739a-4835-8732-4a474fce69a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:39:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:54Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96e0cb36697f namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: f493fe0a-d44b-411a-b19b-251ce215ec64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmxbr namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 8867abea-f006-493a-9925-b75b73ffca4d kind: Event lastTimestamp: "2026-07-02T21:39:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:54Z" name: isvc-autogluon-v2-predictor-67b849d456-bmxbr.18be96e0cb3708d0 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: c993aedf-6941-4f61-92c1-846c88e31cca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "18964" uid: b6c306da-6a18-4d02-b427-fda390948c02 kind: Event lastTimestamp: "2026-07-02T21:37:44Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-bmxbr' metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:37:44Z" name: isvc-autogluon-v2-predictor-67b849d456.18be96c27c6f5f61 namespace: kserve-ci-e2e-test resourceVersion: "18974" uid: 63c4e89f-15d0-4dfb-84de-a93193b79842 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:37:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18961" uid: 149284da-9824-4fa4-9d9a-c6c7a3e7c81f kind: Event lastTimestamp: "2026-07-02T21:37:44Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:37:44Z" name: isvc-autogluon-v2-predictor.18be96c277625f80 namespace: kserve-ci-e2e-test resourceVersion: "18965" uid: 06cd708a-3bcb-499f-8330-70b4f8fcb1ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: 08f68c74-816a-4cdf-a154-e023c47d4542 kind: Event lastTimestamp: "2026-07-02T21:38:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:38:44Z" name: isvc-autogluon-v2-predictor.18be96c5f639f555 namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: 8fea5d30-28af-481f-a32a-4afbfb3def05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: 08f68c74-816a-4cdf-a154-e023c47d4542 kind: Event lastTimestamp: "2026-07-02T21:38:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:38:44Z" name: isvc-autogluon-v2-predictor.18be96c5f6a35aa5 namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: 1d6e0a06-34e4-412b-989f-72c4bb288c68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: 08f68c74-816a-4cdf-a154-e023c47d4542 kind: Event lastTimestamp: "2026-07-02T21:39:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:44Z" name: isvc-autogluon-v2-predictor.18be96d3f31ee76d namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 9040053e-8af1-482d-8665-e22fc13527a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: 08f68c74-816a-4cdf-a154-e023c47d4542 kind: Event lastTimestamp: "2026-07-02T21:39:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:44Z" name: isvc-autogluon-v2-predictor.18be96d3f3884835 namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 952f0624-a43c-48b2-a308-a472f630fd4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:39:59.888242Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:39:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e1ffc5b7ad namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 5e979e18-415b-4ace-b7d4-3016b4ac9c1e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:00Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e21971e53d namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e98427f9-cd75-409b-b7d6-47b8827c75d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e21ae2f19a namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: 74c62b27-22dd-4cae-822d-79335b46695d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e21d68305f namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: a62964c9-a0e6-4752-92e8-e56be897105b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e21e1b9fe9 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 8a99be86-d707-4c0e-9d2c-97275ba528c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e1cff5c8 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 229424ec-8e8b-46cf-aaff-8524ae5a038d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e52f21a6 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 7e485a80-6b50-4082-9521-558b36b9ff38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e5aa58b9 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 07cedb91-7790-4e96-b2e3-244afe848a63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e5b57d78 namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: 82a82749-1868-4691-907b-154eedb709ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e8855211 namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: 8627fac8-4737-4fa2-b0c6-1c1e2b20c191 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:40:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:20Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96e6e8fb23e8 namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: 1bc23e22-0612-4ef7-9c05-8113852d28e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96fdf892d45f namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 17c6dcfd-18ab-407a-bea0-9969b69acddc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7fb6ff88-e1d5-43ba-a91a-5ea0171ffdde kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7.18be96fdf8935444 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 2e71be61-b332-430f-a92a-304d5074179e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: fc9a099e-b538-4dec-9e26-33c42c26042c kind: Event lastTimestamp: "2026-07-02T21:39:59Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-xkpx7' metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18be96e1ff54df10 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: cd9b88f7-81e3-416d-882b-47b1aa2e6c25 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: a887585c-187c-4f88-bede-b732245608d6 kind: Event lastTimestamp: "2026-07-02T21:39:59Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:59Z" name: isvc-autogluon-v2-variants-predictor.18be96e1fadfbdf3 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 92b8ac65-945a-4e49-ba3e-c60ef033570a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: fe6ed71d-143e-4596-a707-1a2943dc15fa kind: Event lastTimestamp: "2026-07-02T21:40:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:44Z" name: isvc-autogluon-v2-variants-predictor.18be96e5799fbc45 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 07d2b6c4-3cc5-4715-8124-3966ac23fa09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: fe6ed71d-143e-4596-a707-1a2943dc15fa kind: Event lastTimestamp: "2026-07-02T21:40:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:44Z" name: isvc-autogluon-v2-variants-predictor.18be96e57a092df6 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 3fc29c71-f23d-4805-8bbd-2c8637f0e284 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: fe6ed71d-143e-4596-a707-1a2943dc15fa kind: Event lastTimestamp: "2026-07-02T21:41:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:59Z" name: isvc-autogluon-v2-variants-predictor.18be96eff743d612 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 5a52c101-6af1-4f4e-ae0b-8384d068a383 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: fe6ed71d-143e-4596-a707-1a2943dc15fa kind: Event lastTimestamp: "2026-07-02T21:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:59Z" name: isvc-autogluon-v2-variants-predictor.18be96eff7a6214c namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: a562cd46-8c04-46bd-8696-4b632d08ac21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: dcb3f956-dd38-45a8-b544-7badf15d7884 kind: Event lastTimestamp: "2026-07-02T21:41:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:56Z" name: isvc-autogluon-v2-variants.18be96e1fae9937a namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: b065e93c-9df1-41c1-b39e-a9e8cdc26fbd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: dcb3f956-dd38-45a8-b544-7badf15d7884 kind: Event lastTimestamp: "2026-07-02T21:39:59Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:59Z" name: isvc-autogluon-v2-variants.18be96e1fb67fa0f namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: 2bd022ef-4b28-48a0-8b2e-2902f9e2b7c0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: dcb3f956-dd38-45a8-b544-7badf15d7884 kind: Event lastTimestamp: "2026-07-02T21:39:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:59Z" name: isvc-autogluon-v2-variants.18be96e1fb683f4e namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 19d69535-3169-45de-89f4-06a55cd150ea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: dcb3f956-dd38-45a8-b544-7badf15d7884 kind: Event lastTimestamp: "2026-07-02T21:41:56Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-02T21:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:56Z" name: isvc-autogluon-v2-variants.18be96fd0ee83a92 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: ce86dc36-7019-40b4-96b5-3cca2fc74637 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:37:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 5180fff9-9d42-49bf-8c8d-21b408978977 kind: Event lastTimestamp: "2026-07-02T21:39:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:53Z" name: isvc-autogluon-v2.18be96c2776b1f97 namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 84765f37-25b1-432f-b626-4f2ee4689b5d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:37:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 5180fff9-9d42-49bf-8c8d-21b408978977 kind: Event lastTimestamp: "2026-07-02T21:37:44Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:37:44Z" name: isvc-autogluon-v2.18be96c2780587ff namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: a1ba790c-1acc-4b6e-82dd-c43f013f9bef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:37:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 5180fff9-9d42-49bf-8c8d-21b408978977 kind: Event lastTimestamp: "2026-07-02T21:37:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:37:44Z" name: isvc-autogluon-v2.18be96c27805cbcb namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: 8710514d-24f1-4a75-b459-4b3ec29a598b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 5180fff9-9d42-49bf-8c8d-21b408978977 kind: Event lastTimestamp: "2026-07-02T21:39:53Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-02T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:53Z" name: isvc-autogluon-v2.18be96e06ca62f71 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: effe9e05-53a3-437a-be9e-3236539a96b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:53.768320Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-757779fc66-gmt6h to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:53Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ec90bb09f namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 0e07d0a4-6693-4689-9f80-56c9fb538da7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:54Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:54Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ee2d805d7 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: f56258c1-454e-43f8-95d0-2119220e86f3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:54Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ee48f168a namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: 761705cb-be1a-4328-bf91-f492cd96e433 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:54Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ee76c677d namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 019d4f66-160d-4a26-8326-2bfaf24c822f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:54Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ee7dd0cf5 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 2bd0b754-db8b-4676-a49c-d7661b0f0947 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ff2ca148b namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: cfa38115-0344-4f86-a3c6-5c45f2a2ee8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ff69474d1 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 07a259ae-f495-42a3-ad32-878fa91d094b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ff714226c namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: a415648b-1b0b-4d70-9218-5c3b16c8b7be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ff71fef09 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 82059b69-b318-4d78-bdb8-d9bc7e5871f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ffa1a3209 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 2218c4af-c9ef-48b1-a554-1772d11b5d86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ffa8d0685 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: d116c499-3118-482d-ab34-ae04108c87dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ffa96fbc0 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 96847b22-9adf-43ce-9116-e69c85746364 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ffd7ded61 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 55083124-006b-400d-8486-22e9478ea98b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:32:58Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T21:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:58Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be967ffe014a0f namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: b3c112c0-e3c3-4e85-ad29-cfe0b05e091a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:32:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:40Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be96802edf725c namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 0e47c862-a911-484f-b9c3-696bc56feed2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:32:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:32:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:30Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be96802ee8cfb0 namespace: kserve-ci-e2e-test resourceVersion: "17549" uid: b15eb4a6-0d7f-497f-be4c-932237078a54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be9694f9818c75 namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: dd99e42d-bcb5-4a29-af36-bd4101fbc6fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be9694f9824c09 namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: fcbf223b-51cf-4adc-9a77-455ebd89598f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be9694f98299f0 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 82ddddb1-f105-46e3-be3d-9261ee4a7e34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-gmt6h namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: d00dbfb7-153a-4f77-bdd2-0fba5dbdadbd kind: Event lastTimestamp: "2026-07-02T21:34:35Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:35Z" name: isvc-logger-predictor-757779fc66-gmt6h.18be96955f4e7c3e namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: b7af288c-ff4f-4f73-abdc-198c071842d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-757779fc66 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: a399dfe1-dcb6-4759-bee4-1f06859816a1 kind: Event lastTimestamp: "2026-07-02T21:32:53Z" message: 'Created pod: isvc-logger-predictor-757779fc66-gmt6h' metadata: creationTimestamp: "2026-07-02T21:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:53Z" name: isvc-logger-predictor-757779fc66.18be967ec889f9c6 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 8433f6c0-2c04-47f9-9cb5-d09a806b2d1f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: bceaf1cc-09f7-4421-99c0-99b7564ce5c4 kind: Event lastTimestamp: "2026-07-02T21:32:53Z" message: Scaled up replica set isvc-logger-predictor-757779fc66 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:53Z" name: isvc-logger-predictor.18be967ec41530f5 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 16fe2f28-2fb2-4b0b-be95-2e7756d45802 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:33:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 46db3f0b-996b-44c7-9e5e-c904bdd5147a kind: Event lastTimestamp: "2026-07-02T21:33:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:33:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:38Z" name: isvc-logger-predictor.18be96824294be79 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 3b5b2eaf-0954-44bb-b0d5-4f0baf5a5b1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:33:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 46db3f0b-996b-44c7-9e5e-c904bdd5147a kind: Event lastTimestamp: "2026-07-02T21:33:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:33:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:38Z" name: isvc-logger-predictor.18be968242fa6ed9 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 77e99d06-854d-4ea8-b1b6-5a4ed8fb99c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 46db3f0b-996b-44c7-9e5e-c904bdd5147a kind: Event lastTimestamp: "2026-07-02T21:34:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:23Z" name: isvc-logger-predictor.18be968cbfe78f4b namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: a23d7e0f-5928-4dc3-803c-0a9d96e2631e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 46db3f0b-996b-44c7-9e5e-c904bdd5147a kind: Event lastTimestamp: "2026-07-02T21:34:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:23Z" name: isvc-logger-predictor.18be968cc057b95b namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 6dd105e3-6838-48b2-a7a7-193ef51c7bb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:32:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 7304052c-a471-484e-b338-994cae83d885 kind: Event lastTimestamp: "2026-07-02T21:34:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:20Z" name: isvc-logger.18be967ec4273d44 namespace: kserve-ci-e2e-test resourceVersion: "17403" uid: 08b429d3-0fcc-4da7-9368-fe2849415d99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 7304052c-a471-484e-b338-994cae83d885 kind: Event lastTimestamp: "2026-07-02T21:32:53Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:53Z" name: isvc-logger.18be967ec48f956b namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: cdc2ca8f-65fd-4e28-9d0d-c79e739f3a1e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17400" uid: 7304052c-a471-484e-b338-994cae83d885 kind: Event lastTimestamp: "2026-07-02T21:34:20Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-02T21:34:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:20Z" name: isvc-logger.18be9692e40786b7 namespace: kserve-ci-e2e-test resourceVersion: "17401" uid: 9dcc1bae-edd4-4a19-8246-b2e9e390184d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:30:58.567347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be9663f6888d1f namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: a9faef7b-e893-400b-88e8-e4a14d0869f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:30:59Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:30:59Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966410696d48 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: bafb27d4-5391-44ce-98aa-4db573861de0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:30:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:59Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966411e78af6 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 9d458f32-4231-47fa-b91a-064a660fc60b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:30:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:59Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be9664157652f3 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: ebebab76-8410-4429-8d04-095add97158a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:30:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:59Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966415e77735 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 746ff0cc-23c9-4c19-8f12-2d678ccbadf9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be96650bac70d9 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: a91c34ee-28a6-4a8b-8622-a95489710856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be96650ed6b64c namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 69413f8f-e672-4ef7-898d-ce4130357b12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be96650f4ad2aa namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 349b4664-f428-4e93-964e-2a2435371088 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be96650f5574bf namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: d6b31fa9-8914-4aab-8131-d84b3d0a81c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be9665123a1e0c namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 1b462307-a195-4c59-be4a-008e40534dc3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966512b9ca05 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: c280e94e-fe16-4d04-96d3-6f1869070f8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966512c5a36b namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 5e2f4333-c638-436a-86df-09246a9678d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966515abfe1b namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: d991f1e4-f27d-44cb-b161-cf830b80332c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be9665161a2d48 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: ab2f53ea-61b7-4a92-a4ae-bcc12bed3d9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:50Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966547eb7544 namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 7cba3186-36d7-4fd6-9945-bb89d5df6a3e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:40Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be966547f4fad5 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: dd8afd97-a7aa-4a6d-8736-4176319d2b2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:33Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be967a203117d8 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: c7dbc651-375b-45af-b6c9-6693b90c2201 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:33Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be967a203157ee namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: a117ca05-19ff-4a74-b05d-4383dc5bc3d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:33Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be967a20317b56 namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: a1d6af5d-c13c-4e63-8e13-ed1c4fa1bb80 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: cf3e2e4f-c62f-49c6-a622-c4d9c7d3d925 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:50Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f.18be967a7836b53e namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 58e38e67-2e65-4416-a51a-4c8ae68bc44c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-684888dfc6 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: fb9a7843-142c-4a65-939b-f5445400d85d kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-684888dfc6-ghs4f' metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6.18be9663f608f1af namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: f26a38c0-fd6e-477a-9b0f-703ccb4d3b35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 56c59050-c131-41eb-8b08-936707d9d5be kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-684888dfc6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-custom-predictor.18be9663ed42f51a namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 79ed22f4-a8ae-4703-bd73-490198b4bd20 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: db2051b6-63e3-410f-85cd-f6fc92f3dfcd kind: Event lastTimestamp: "2026-07-02T21:31:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:28Z" name: isvc-sklearn-batcher-custom-predictor.18be96676bc2ed3d namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: eacb35c7-285c-4677-9208-54092fd0706e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: db2051b6-63e3-410f-85cd-f6fc92f3dfcd kind: Event lastTimestamp: "2026-07-02T21:31:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:28Z" name: isvc-sklearn-batcher-custom-predictor.18be96676c441ae0 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 99a40904-a498-40ab-af2d-ab2db75e29ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:31:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: db2051b6-63e3-410f-85cd-f6fc92f3dfcd kind: Event lastTimestamp: "2026-07-02T21:32:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:28Z" name: isvc-sklearn-batcher-custom-predictor.18be966e6a02b229 namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 32fb5a8b-27c4-417c-9d53-f72d5f683ca9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:31:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: db2051b6-63e3-410f-85cd-f6fc92f3dfcd kind: Event lastTimestamp: "2026-07-02T21:32:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:28Z" name: isvc-sklearn-batcher-custom-predictor.18be966e6a645836 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 3f293759-424e-4800-ad48-249280ae5dbc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: f7285b46-ec93-4850-9d0c-2f7f588adbd7 kind: Event lastTimestamp: "2026-07-02T21:32:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:20Z" name: isvc-sklearn-batcher-custom.18be9663ed497ad8 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 2da3cb19-6811-43c0-bb0c-413b447a816c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: f7285b46-ec93-4850-9d0c-2f7f588adbd7 kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-custom.18be9663edc923ec namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 62709c87-4cdf-4f69-a516-0b865363b82e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: f7285b46-ec93-4850-9d0c-2f7f588adbd7 kind: Event lastTimestamp: "2026-07-02T21:32:20Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-02T21:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:20Z" name: isvc-sklearn-batcher-custom.18be967701291b8e namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 28cef8ad-6171-4efd-80e5-e33fcbf5de78 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:03.369849Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-956c77b89-x84t4 to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be9649243a136e namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 8ae09c17-3a92-42d5-b449-7c3729cf0831 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be96493df8ccce namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 0602ac6e-607b-4b00-822f-5619dded405b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be96493fa419b6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: a88f61a4-5796-4d35-965d-a08503921029 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be9649427e079d namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 7c650768-c443-4a75-8a65-df4c750a4550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964942f79e60 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: f0e2755a-9feb-42ce-8d17-774e45960592 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:08Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964a5f2f1ccc namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: def2502f-2f04-4d92-ba09-0e9dd2fed693 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 13.492s (13.492s including waiting). Image size: 1555748775 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:22Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964d83695272 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: ab6fd91a-759b-4c31-ac04-3dd9f5d26af7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:22Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964d86c0e46a namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 0de70d7c-7566-4e53-895b-8423d56b237b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:22Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964d873b1a8f namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: c1dcf328-4ed7-48e1-9ebe-cac8fb66ede7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:22Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964d8749713a namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 048de206-4f64-49ff-9a6a-0e04556fdfec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:24Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.179s (2.179s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:24Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e092d68e1 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 795bb992-dea5-4316-9509-1684f46f5c3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:24Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e0c344576 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 5334a965-5653-4a81-8fc7-23986d1ac3d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:24Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e0cacafd2 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: ca3a16a7-b354-4ded-9b47-ed06ead4172b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:24Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" metadata: creationTimestamp: "2026-07-02T21:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:24Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e0cbb4567 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: ce492838-fc82-41b3-8bfd-ac07b787c60f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" in 2.1s (2.1s including waiting). Image size: 239997024 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e89f18fa2 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: aeddc963-c15a-4a91-9bad-6cdb7f03be45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e8ceadfaa namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 9c383f3d-e7b2-4e0a-aa01-c582a9efc3df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e8d62d400 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 152612a3-4272-41cb-90dc-07310d690751 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964e975e1e02 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 33bd692e-1d57-4930-b4d0-46ae69f90cfb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:31:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:03Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be964ed33e3cb9 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 8235b105-fa7d-47ee-a9ab-618725a36bcc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be9663f918213e namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 92e2c74b-2c77-42fc-bbc0-b94667af4375 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be9663f9186714 namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 94e6c93a-1dec-4a3e-9175-8d15374d6e68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:58Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be9663f918f3f8 namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 582e9dde-2680-4705-b0e7-68d8f682c491 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-x84t4 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d3b7deba-0535-4464-adde-8deb987c6539 kind: Event lastTimestamp: "2026-07-02T21:31:13Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:13Z" name: isvc-sklearn-batcher-predictor-956c77b89-x84t4.18be966403860439 namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: bb548704-5b91-4ffd-9a55-74f190f9e784 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-956c77b89 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 2561c614-34a3-41da-858a-ec7bb9444336 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: 'Created pod: isvc-sklearn-batcher-predictor-956c77b89-x84t4' metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor-956c77b89.18be964923b9f864 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 429e2b46-0edb-4e5b-8806-9a8b6c0e6c6e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 2c0bb998-6a2d-49d7-99ba-571e5dd21a02 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-956c77b89 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher-predictor.18be96491b5aef63 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 1bf5ce19-0bcd-4113-bd5f-14245db6cce5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: cdc3c789-aae0-4a1c-8e78-3cacd0fb7574 kind: Event lastTimestamp: "2026-07-02T21:30:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:03Z" name: isvc-sklearn-batcher-predictor.18be964ca0ed631a namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: d87fb5a2-7a06-4c9d-866d-273e7d4670de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: cdc3c789-aae0-4a1c-8e78-3cacd0fb7574 kind: Event lastTimestamp: "2026-07-02T21:30:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:03Z" name: isvc-sklearn-batcher-predictor.18be964ca1575033 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: c2342a4b-5dbf-43d5-964c-af5533d84ba2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: cdc3c789-aae0-4a1c-8e78-3cacd0fb7574 kind: Event lastTimestamp: "2026-07-02T21:30:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:48Z" name: isvc-sklearn-batcher-predictor.18be965a9d713ac1 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 6386c8df-06fb-4a3b-bcaf-0ee7eeb0afed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: cdc3c789-aae0-4a1c-8e78-3cacd0fb7574 kind: Event lastTimestamp: "2026-07-02T21:30:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:48Z" name: isvc-sklearn-batcher-predictor.18be965a9dd6e28b namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: cd4a86fa-a25a-48f9-a8c4-3fbcc4054232 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: f467c1ec-b25c-40d7-9071-a48ec444cf37 kind: Event lastTimestamp: "2026-07-02T21:30:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:43Z" name: isvc-sklearn-batcher.18be964921be5f89 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 9c52ee02-188b-4122-8e94-eae1467e8883 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: f467c1ec-b25c-40d7-9071-a48ec444cf37 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher.18be9649221264f2 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 9e2f56d7-a739-4220-ab5c-c1538c0cd3ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: f467c1ec-b25c-40d7-9071-a48ec444cf37 kind: Event lastTimestamp: "2026-07-02T21:29:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:29:03Z" name: isvc-sklearn-batcher.18be96492212c9d4 namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 483f4755-ce28-4edb-95ec-fc80c5f1a270 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: f467c1ec-b25c-40d7-9071-a48ec444cf37 kind: Event lastTimestamp: "2026-07-02T21:30:43Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-02T21:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:43Z" name: isvc-sklearn-batcher.18be96608b9e2f47 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: be2fb9da-48d9-4120-bb7c-2ca467219267 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:33.709162Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-bt87v to ip-10-0-134-124.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:33Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a1d6d3765 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: b210b5a6-a374-466a-b476-50be2ff829bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584845dcb9-lr8hb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:34Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:34Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:34Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a37366857 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: d34ce8e5-d7e2-47b1-86f2-8e9654a078b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:34Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T21:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:34Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a388221e1 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 2147ec18-be51-4a2d-bcac-f85d8ed57513 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.385s (1.385s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a8b168621 namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 279f99b1-58ff-49fb-86c4-f67b351ab75e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a8e613ed4 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 30af497f-5e4f-4f8f-82db-7de3bfdfe23b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a8edb3b99 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 9047b9a4-ea80-4adc-9bcd-758e7df659a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a8ee55d7b namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 7960df7b-678d-436c-a70c-33eeb11a24ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a91fb64a6 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: d2525902-9c7a-450c-ab47-f0adfc374b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be967a928a570f namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 0eab565b-d5c6-4552-8b1c-4f59acd7571f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be9694fd245b19 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 751e4b60-5eb3-45eb-a513-89f5d6f64fde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-bt87v namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 37135ac3-b594-414c-8135-3a462e132e1f kind: Event lastTimestamp: "2026-07-02T21:34:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:29Z" name: message-dumper-predictor-6575bcfdfd-bt87v.18be9694fd25053d namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: 4fdcb9d9-ff47-458a-8d86-ea976d6e6ebc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-124.ec2.internal source: component: kubelet host: ip-10-0-134-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 8dc6c0f4-574f-4d2f-8741-070deb0c2376 kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-bt87v' metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:33Z" name: message-dumper-predictor-6575bcfdfd.18be967a1c96ab1c namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: c4ea2792-948f-4295-8493-c6cc7b8261e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: fb09efb8-f15d-4bd6-8cfb-f5031576a0bf kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:33Z" name: message-dumper-predictor.18be967a18a3eefd namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 0d2ec60b-28ff-4f0b-b34c-bac0e179677e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 687554a9-bd88-416c-86bc-43cf23e51340 kind: Event lastTimestamp: "2026-07-02T21:33:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:03Z" name: message-dumper-predictor.18be967d973a60d1 namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 454d5600-4a98-473a-a7f6-790c184337bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 687554a9-bd88-416c-86bc-43cf23e51340 kind: Event lastTimestamp: "2026-07-02T21:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:03Z" name: message-dumper-predictor.18be967d979e19fc namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: a96f8468-1b1a-45ef-be13-191bd06906f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 6fc9d5f9-4a68-43e6-8aca-d5d670b593ae kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:43Z" name: message-dumper.18be967a18aa9d5b namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 21ad71e8-d69a-48ff-a583-1e0c5b423665 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 6fc9d5f9-4a68-43e6-8aca-d5d670b593ae kind: Event lastTimestamp: "2026-07-02T21:32:33Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:33Z" name: message-dumper.18be967a194ee784 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 3ebe2642-50af-407f-a6fe-8170c0ef8bfb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 6fc9d5f9-4a68-43e6-8aca-d5d670b593ae kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:43Z" name: message-dumper.18be967c747c2d6e namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 230fbe06-4592-47bb-a82e-d3eddfda31f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22413"