--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:41:10.883431Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22995" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:41:10Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd144cca6515c namespace: kserve-ci-e2e-test resourceVersion: "22999" uid: 2ae8a8d9-6ee9-4f64-a8ed-c04a3c1fca76 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-ts-v1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd144d8f0c369 namespace: kserve-ci-e2e-test resourceVersion: "23010" uid: d8aedfe6-c7d9-4048-b3ad-988ec5c5cb4c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:11Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:41:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14509d5eebd namespace: kserve-ci-e2e-test resourceVersion: "23018" uid: 3ded737d-b6fe-4e65-ae8d-443bef9eeacb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd1450b44e395 namespace: kserve-ci-e2e-test resourceVersion: "23020" uid: c4811a88-2172-4ca3-ba17-dfd71e0ddeb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd1450df44b84 namespace: kserve-ci-e2e-test resourceVersion: "23021" uid: 2c901ec7-2fe2-4626-adff-c6ba75314b27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:11Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd1450e646f94 namespace: kserve-ci-e2e-test resourceVersion: "23022" uid: 2f7a49ed-fae3-4169-bb1e-a42662b05d75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd1472c613079 namespace: kserve-ci-e2e-test resourceVersion: "23093" uid: 5aa3fc65-ad02-404d-9819-6e96071f2c14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd1473034afb8 namespace: kserve-ci-e2e-test resourceVersion: "23095" uid: 84c34cdc-be18-4a4b-b31d-67c5896666c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14730af9e16 namespace: kserve-ci-e2e-test resourceVersion: "23098" uid: c3398817-688c-41a5-9730-ca4ca9d95563 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14730ba7519 namespace: kserve-ci-e2e-test resourceVersion: "23099" uid: d81fe518-a7c2-45ac-8081-8dc953186c69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14734225f69 namespace: kserve-ci-e2e-test resourceVersion: "23101" uid: 3fc9dcb0-35fe-4661-9e19-89c7ff01d154 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:41:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:21Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd147349b470d namespace: kserve-ci-e2e-test resourceVersion: "23102" uid: ef1c897e-e72d-4dae-a21d-a327d1103c57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd147a45b3ea8 namespace: kserve-ci-e2e-test resourceVersion: "23163" uid: 74b3c868-8f4c-4c1b-a411-a564caef2207 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:40Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14bcbc8c9f5 namespace: kserve-ci-e2e-test resourceVersion: "23269" uid: 75e4232e-460e-4bdf-9470-a3b8308ca093 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 4a041dab-85bd-4b08-86dc-9d078ba1cf60 kind: Event lastTimestamp: "2026-07-06T21:41:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:40Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989.18bfd14bcbc974a7 namespace: kserve-ci-e2e-test resourceVersion: "23270" uid: dc0623c5-6ed8-46af-87ff-eca4f750f588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "22985" uid: 4fcb7f81-da0e-409d-a801-6a01840b83bb kind: Event lastTimestamp: "2026-07-06T21:41:10Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-qk989' metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:10Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18bfd144cc4048a7 namespace: kserve-ci-e2e-test resourceVersion: "22996" uid: ac4c6ca1-1885-4aa6-bc1a-6589777e19af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22983" uid: 46cf34f5-c52f-4553-bc7c-587de58b9618 kind: Event lastTimestamp: "2026-07-06T21:41:10Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:10Z" name: isvc-autogluon-ts-v1-predictor.18bfd144c8163ab5 namespace: kserve-ci-e2e-test resourceVersion: "22986" uid: d64b09d6-e369-42e0-8ac1-13287a2f0a5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:41:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22984" uid: b61db058-a885-4bda-ab41-b0d4338ff1f8 kind: Event lastTimestamp: "2026-07-06T21:41:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:40Z" name: isvc-autogluon-ts-v1-predictor.18bfd14846c93947 namespace: kserve-ci-e2e-test resourceVersion: "23253" uid: d64c124b-bdb9-4972-8dca-8b655f9fa90e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:41:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22984" uid: b61db058-a885-4bda-ab41-b0d4338ff1f8 kind: Event lastTimestamp: "2026-07-06T21:41:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:40Z" name: isvc-autogluon-ts-v1-predictor.18bfd148472c824d namespace: kserve-ci-e2e-test resourceVersion: "23254" uid: 61414025-7898-4792-8b1e-0a1883194498 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:41:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 105a342a-4543-4db3-879a-4b04040c8eab kind: Event lastTimestamp: "2026-07-06T21:41:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:39Z" name: isvc-autogluon-ts-v1.18bfd144c82041d2 namespace: kserve-ci-e2e-test resourceVersion: "23238" uid: 30788683-3c32-4ebf-b0cf-05766e515300 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 105a342a-4543-4db3-879a-4b04040c8eab kind: Event lastTimestamp: "2026-07-06T21:41:10Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:10Z" name: isvc-autogluon-ts-v1.18bfd144c8885adf namespace: kserve-ci-e2e-test resourceVersion: "22990" uid: 03187684-dad4-40ae-be02-8106d6f1ebde reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 105a342a-4543-4db3-879a-4b04040c8eab kind: Event lastTimestamp: "2026-07-06T21:41:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:10Z" name: isvc-autogluon-ts-v1.18bfd144c8889c3a namespace: kserve-ci-e2e-test resourceVersion: "22991" uid: d21cc504-d92f-4648-b588-939a1e7e1ebe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "23236" uid: 105a342a-4543-4db3-879a-4b04040c8eab kind: Event lastTimestamp: "2026-07-06T21:41:39Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-06T21:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:39Z" name: isvc-autogluon-ts-v1.18bfd14b63b28fa2 namespace: kserve-ci-e2e-test resourceVersion: "23237" uid: 4460487c-9870-46fb-b828-64841611ecb7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:31:39.578971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:31:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0bfc8400d24 namespace: kserve-ci-e2e-test resourceVersion: "18679" uid: 83670f07-206e-4776-a72f-cb6eac44d72d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:31:40Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0bfe1cd450a namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: 6b382a60-f3ee-4390-8d24-7c1435c29cb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:31:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0bfe3393e27 namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: a93899d0-139d-4357-9586-1b88af6c9955 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:31:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0bfe5e27745 namespace: kserve-ci-e2e-test resourceVersion: "18704" uid: 36971f96-b31f-4aac-9e88-8b3d662d0d24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:31:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:40Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0bfe652b0ec namespace: kserve-ci-e2e-test resourceVersion: "18705" uid: 3aa3f5a5-668b-48b9-981f-149788894421 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:32:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:32:10Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-06T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:32:10Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0c6e146d35c namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 74df8950-bad0-46f1-a8bb-cf56047490c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:21Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m11.851s (2m11.851s including waiting). Image size: 8939170586 bytes.' metadata: creationTimestamp: "2026-07-06T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:21Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e59438ce1b namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: 46e9f08d-00c5-4fc8-aef7-011294fcc9d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e59a356222 namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: 39243da9-165e-4c1e-b1ac-1696521fac07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e59c75f0a1 namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: 99d3d8aa-7536-4e64-b883-6c52026c3ae4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e59c83004b namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: db165f1f-dba2-4269-b8e5-82746b4b572b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e5a4fe0efe namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: 111b2601-3735-475c-9180-c0f5cb43798a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e5a8a7e53b namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: c7aa83da-43a0-4dda-81f2-d1eb650391bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0e5f756de05 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 19358e5f-afc4-4525-90ac-d6857e0dddc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0ec0c56726f namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: e588dca0-ba32-4d42-89cd-9254096a0d4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 8379cd1e-92b2-47d2-9e9d-cc6861e44030 kind: Event lastTimestamp: "2026-07-06T21:34:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w.18bfd0ec0c56f593 namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 65caabc7-b250-4a15-9cb5-3d9b2e02dbe1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: af01ffff-937e-47d2-acc1-83de41dc134f kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-lrx4w' metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:31:39Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18bfd0bfc7c898de namespace: kserve-ci-e2e-test resourceVersion: "18672" uid: 61a9eb56-cd70-4c1d-8192-4b94c7ec6ac9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18635" uid: 9e1d258a-6fa6-40a5-8774-d0809cb12825 kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:31:39Z" name: isvc-autogluon-v1-predictor.18bfd0bfc2efa01e namespace: kserve-ci-e2e-test resourceVersion: "18638" uid: 66a40d53-5974-44ae-9176-3f9cc9f08253 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:31:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18636" uid: 89f3b22b-22ae-42db-a97b-4d8cb08a7c60 kind: Event lastTimestamp: "2026-07-06T21:34:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:31:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:34:39Z" name: isvc-autogluon-v1-predictor.18bfd0c341664824 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: efa19efa-e545-4e72-a33d-beefe002a2c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:31:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18636" uid: 89f3b22b-22ae-42db-a97b-4d8cb08a7c60 kind: Event lastTimestamp: "2026-07-06T21:34:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:31:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:34:39Z" name: isvc-autogluon-v1-predictor.18bfd0c341c39c87 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 70e66bca-129d-4743-9554-9f9a3fc72df2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: d695d773-5905-4098-a2c9-11785251ce8c kind: Event lastTimestamp: "2026-07-06T21:34:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:34:39Z" name: isvc-autogluon-v1.18bfd0bfc2f5326a namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 6b5c678b-c8ca-4b2e-92f8-63e20d4e724c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: d695d773-5905-4098-a2c9-11785251ce8c kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:31:39Z" name: isvc-autogluon-v1.18bfd0bfc3934fdf namespace: kserve-ci-e2e-test resourceVersion: "18641" uid: daaa93bd-1aa5-4d1b-a771-84cec56e4027 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: d695d773-5905-4098-a2c9-11785251ce8c kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:31:39Z" name: isvc-autogluon-v1.18bfd0bfc393ae58 namespace: kserve-ci-e2e-test resourceVersion: "18643" uid: 38b089a5-0e35-4183-ba2b-f0094a8d6b7c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: d695d773-5905-4098-a2c9-11785251ce8c kind: Event lastTimestamp: "2026-07-06T21:34:39Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-06T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:34:39Z" name: isvc-autogluon-v1.18bfd0e9b64bd67b namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: e0259fea-0a87-402c-a039-aef6328479c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:39:05.579516Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:39:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd1279ff44fcb namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 1dcc40ca-a317-404c-843d-667f033e4b8b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:06Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:39:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd127b98c7cfb namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: fc4485f4-6af4-4796-8f1c-85b16178249d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd127baeb48ac namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: ab8b1128-a7a4-4907-a1ee-1937e23a7656 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd127bdf74332 namespace: kserve-ci-e2e-test resourceVersion: "22060" uid: 738a9a5b-5a17-4f32-a91f-a2d324183640 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:06Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd127be69798e namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 35e89d04-799d-4f75-bcec-d375d23c1fe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e2bb6fbf9 namespace: kserve-ci-e2e-test resourceVersion: "22255" uid: 3b993cdb-5f32-407e-8db6-73fb72ac1d6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e2eed112c namespace: kserve-ci-e2e-test resourceVersion: "22259" uid: 81867f3a-1805-4910-9bf6-79d0ef4a02b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e2f670f25 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: f7f87d6b-7cf9-415a-9394-9b4de9fa1def reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e2f724e5c namespace: kserve-ci-e2e-test resourceVersion: "22261" uid: 08e1732d-69c0-449a-8030-a45a1ad296d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e32816156 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: a2aaa5fb-d50c-406c-bb91-3258c8e0aae8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:39:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:33Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd12e32f64ccf namespace: kserve-ci-e2e-test resourceVersion: "22265" uid: f8c1f1e6-9403-4139-be4b-372347ab147f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:41:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd1439912902b namespace: kserve-ci-e2e-test resourceVersion: "22936" uid: 8fa1587f-2f68-44d2-9dd4-525c590c6c28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 6063bdc9-8fc4-4be0-a2b3-4667e26ed9c2 kind: Event lastTimestamp: "2026-07-06T21:41:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:41:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r.18bfd143991317db namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: d7245bb8-6034-45fc-8596-446f4dd8a36a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: b8ffc616-a32e-4d13-a9a2-51dcccfe7d31 kind: Event lastTimestamp: "2026-07-06T21:39:05Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-tqc9r' metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:39:05Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18bfd1279f8e01ca namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: f90e1246-7862-4b83-a3cc-5087dc14b99e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: e06f56de-f5c8-496d-949e-ec5510fd6c4f kind: Event lastTimestamp: "2026-07-06T21:39:05Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:39:05Z" name: isvc-autogluon-v2-noslash-predictor.18bfd1279b25eed0 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: c38f04bb-29f9-44cd-9fc5-212e0606aa30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:39:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 24c49921-5659-4e96-9968-e7e7bafefff5 kind: Event lastTimestamp: "2026-07-06T21:39:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:39:50Z" name: isvc-autogluon-v2-noslash-predictor.18bfd12b19a43b0b namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 6d09d54e-5eff-4689-ad76-30d35c974e08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:39:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 24c49921-5659-4e96-9968-e7e7bafefff5 kind: Event lastTimestamp: "2026-07-06T21:39:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:39:50Z" name: isvc-autogluon-v2-noslash-predictor.18bfd12b1a04dfeb namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 06449520-a114-4bb8-97b9-dfe716c9b8c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:40:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: 24c49921-5659-4e96-9968-e7e7bafefff5 kind: Event lastTimestamp: "2026-07-06T21:41:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:05Z" name: isvc-autogluon-v2-noslash-predictor.18bfd13596e8f0f9 namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: db28e7e5-6612-48f0-bbd4-74b702bbdf0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:40:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: 24c49921-5659-4e96-9968-e7e7bafefff5 kind: Event lastTimestamp: "2026-07-06T21:41:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:41:05Z" name: isvc-autogluon-v2-noslash-predictor.18bfd1359755b345 namespace: kserve-ci-e2e-test resourceVersion: "22921" uid: daa3bd2b-c98f-46f9-909e-d80ef2f3155b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T21:39:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 2d15c353-4f91-4dba-a4c3-32ae0ee50b66 kind: Event lastTimestamp: "2026-07-06T21:41:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:05Z" name: isvc-autogluon-v2-noslash.18bfd1279b35ab8f namespace: kserve-ci-e2e-test resourceVersion: "22913" uid: a06a8cef-86d4-400d-8215-e6e4bfa0139f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:39:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 2d15c353-4f91-4dba-a4c3-32ae0ee50b66 kind: Event lastTimestamp: "2026-07-06T21:39:05Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:39:05Z" name: isvc-autogluon-v2-noslash.18bfd1279bb0d1a9 namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: d82d40b7-592b-440e-9409-05eb41a8bc6b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:39:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 2d15c353-4f91-4dba-a4c3-32ae0ee50b66 kind: Event lastTimestamp: "2026-07-06T21:39:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:39:05Z" name: isvc-autogluon-v2-noslash.18bfd1279bb10b66 namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: ff268373-0aab-4c15-9e09-a68166cefbbc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22909" uid: 2d15c353-4f91-4dba-a4c3-32ae0ee50b66 kind: Event lastTimestamp: "2026-07-06T21:41:04Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-06T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:41:04Z" name: isvc-autogluon-v2-noslash.18bfd1436cd7fad9 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 24c4048e-747d-4086-ac88-06f17a4378c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:34:55.046659Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-zmv5r to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:34:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:34:55Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed4b085296 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 02caa100-601c-4e6d-905f-829e87009a6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:34:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:55Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed54d347c0 namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 58d7e454-e87b-4aa8-864b-91130bf886b4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:34:56Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:34:56Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed8bcfdcea namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 2d8072a6-b0b1-494a-838f-bae9dbe65ee5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:34:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:56Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed8db75a99 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 812498ef-aafb-4af4-b23c-e1cfe0a7b6f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:34:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:56Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed93dc37d3 namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: c7fda503-b138-4715-845f-001582eef4e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:34:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:34:56Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0ed9505ad02 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 5a030e84-f11d-4e99-b764-757582a0f248 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d1720237 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: bd5ccaae-36f9-46cc-adcb-7c01bc9a443f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d45bafb5 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: ca335725-ba6b-4230-81bf-17496713809d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d4bc82c1 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: cbd68711-f44a-4f48-9fa8-038155477b36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d4c7be78 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2d790171-9722-43a7-a5b9-daf857bc1b3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d7d488bb namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: c584181a-271e-45af-8b46-00676f8fec73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:35:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:35:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:35:18Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd0f2d83e639f namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 07f23002-b8cb-453d-b8ba-b6f122df3398 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:37:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:05Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd10b9678a994 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: dbb2a555-9f08-418a-84a9-ce238f3652f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zmv5r namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b63a58ca-fa61-4643-b896-b722fb7c8c06 kind: Event lastTimestamp: "2026-07-06T21:37:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:05Z" name: isvc-autogluon-v2-predictor-67b849d456-zmv5r.18bfd10b96793a39 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: db5c675a-f6f9-45c4-8482-72b325c42b5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 06806ff6-2e73-4202-977a-23c82dfbfd1d kind: Event lastTimestamp: "2026-07-06T21:34:55Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-zmv5r' metadata: creationTimestamp: "2026-07-06T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:34:55Z" name: isvc-autogluon-v2-predictor-67b849d456.18bfd0ed4a7b0060 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: 852821f4-3be5-469e-ae32-1862ed1594b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:34:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 676f4a8c-7041-49a3-9b63-1fb4452f26dd kind: Event lastTimestamp: "2026-07-06T21:34:54Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:34:54Z" name: isvc-autogluon-v2-predictor.18bfd0ed45d6c7ad namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 8a7f7732-ae84-480f-9190-983fa6668f64 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:35:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: ed07f1f7-6a31-4a26-9580-b561488505ff kind: Event lastTimestamp: "2026-07-06T21:35:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:35:55Z" name: isvc-autogluon-v2-predictor.18bfd0f0c4bab13b namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 1889ba89-b990-4982-a583-cfc2f9b3a6aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:35:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: ed07f1f7-6a31-4a26-9580-b561488505ff kind: Event lastTimestamp: "2026-07-06T21:35:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:35:55Z" name: isvc-autogluon-v2-predictor.18bfd0f0c52acd59 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: a8273ca5-8f3f-41d5-90aa-b9a8dff2d21b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:35:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: ed07f1f7-6a31-4a26-9580-b561488505ff kind: Event lastTimestamp: "2026-07-06T21:36:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:36:55Z" name: isvc-autogluon-v2-predictor.18bfd0f7c301f1c2 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: a96c27c5-408f-40c2-97e9-23e54830e4cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:35:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: ed07f1f7-6a31-4a26-9580-b561488505ff kind: Event lastTimestamp: "2026-07-06T21:36:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:36:55Z" name: isvc-autogluon-v2-predictor.18bfd0f7c3672c71 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: fa5b3b98-884d-4eea-bf65-493d916e55fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:37:10.297321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10cc899a664 namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 04478a2d-d20e-49be-968f-ff8b5f5f93b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10ce1dff4bf namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 7084391c-3b6d-4894-a8ec-a45599eb8d11 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10ce3575d80 namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: b09d7a99-6535-4f37-beca-8224a64b0430 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10ce5865c6f namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 1ab22ab5-a009-448d-bfe1-3c496aa614d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10ce5eaa9ec namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 3aa2e175-84a6-4557-87a7-e949d83d5fd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e28924051 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: f62e4e1a-1f4e-4de1-a17e-67c60f81d1cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e2bc23da0 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 68f3b521-72d4-42c1-bce2-814d9e896396 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e2c251354 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: ba312c8a-5360-4dc6-a6ee-d47067aee11a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e2c2fdf88 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: e11a3c39-8625-4e0d-a42e-c40d3c46fb46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e2f06a982 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: d0dccc45-4857-40ad-9065-1207c1b34f28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:37:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:37:16Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd10e2f6c6b25 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: af8ac396-5777-436d-9578-e0905bef6def reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:39:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd1266d3458fd namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 84b0461b-0e39-4442-b79c-a0201b39d7b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3be03b3a-1b2f-4173-bf09-e0b2b4f1a8e7 kind: Event lastTimestamp: "2026-07-06T21:39:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:39:00Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm.18bfd1266d34b681 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: c51ef80a-5d58-49bc-a5d7-f54011820101 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 5c15f5a1-fb26-424a-aa10-0fedd2bc2567 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-s7cnm' metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18bfd10cc830e918 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 5b787fe8-a656-45bc-8e7f-807f57d613e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 723045ea-6681-438a-b29c-abafd8d60c5a kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants-predictor.18bfd10cc412e8b4 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 053abac5-bdcc-4d8a-ab29-67becf6d020e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 6a4e8fe8-37b5-484d-b69a-bbe2b56d7cd9 kind: Event lastTimestamp: "2026-07-06T21:37:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:37:40Z" name: isvc-autogluon-v2-variants-predictor.18bfd11042a2a3d4 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: d235be56-cd27-41c6-9792-9f0e4b6062b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 6a4e8fe8-37b5-484d-b69a-bbe2b56d7cd9 kind: Event lastTimestamp: "2026-07-06T21:37:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:37:40Z" name: isvc-autogluon-v2-variants-predictor.18bfd11043026f83 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: f6d6be88-a47c-4d5d-b578-52a21a6ef9ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 6a4e8fe8-37b5-484d-b69a-bbe2b56d7cd9 kind: Event lastTimestamp: "2026-07-06T21:38:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:38:55Z" name: isvc-autogluon-v2-variants-predictor.18bfd11740d0660a namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 0973da91-6650-4a91-bdc6-39180fc75d18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 6a4e8fe8-37b5-484d-b69a-bbe2b56d7cd9 kind: Event lastTimestamp: "2026-07-06T21:38:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:38:55Z" name: isvc-autogluon-v2-variants-predictor.18bfd117413389fd namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 93a9d1a0-8052-471f-8498-8b9334d836ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 013d66bf-e012-403d-9c56-a75a8f149090 kind: Event lastTimestamp: "2026-07-06T21:38:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:38:56Z" name: isvc-autogluon-v2-variants.18bfd10cc41a6689 namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: 0ec0311c-bf87-41f3-8f33-1c8afe51440c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 013d66bf-e012-403d-9c56-a75a8f149090 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants.18bfd10cc4b109f3 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 18f8d5c4-ddd1-4115-a61a-3dcb0a9e9130 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 013d66bf-e012-403d-9c56-a75a8f149090 kind: Event lastTimestamp: "2026-07-06T21:37:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:37:10Z" name: isvc-autogluon-v2-variants.18bfd10cc4b157fa namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 229fae2d-06ce-48b1-b193-53c0fc1e4802 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:38:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 013d66bf-e012-403d-9c56-a75a8f149090 kind: Event lastTimestamp: "2026-07-06T21:38:55Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-06T21:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:38:55Z" name: isvc-autogluon-v2-variants.18bfd12563072aca namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 166b8025-534c-4742-8ee5-ac67f9c94532 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:34:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 52319ba5-1182-4ecf-a657-77f6b5096603 kind: Event lastTimestamp: "2026-07-06T21:36:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:36:56Z" name: isvc-autogluon-v2.18bfd0ed4603f2f9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 9117bc68-0213-4237-9f31-12aa5a93f8aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:34:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 52319ba5-1182-4ecf-a657-77f6b5096603 kind: Event lastTimestamp: "2026-07-06T21:34:55Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:34:55Z" name: isvc-autogluon-v2.18bfd0ed46aac255 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 9fdd9bfa-0067-4fb3-8e2a-35e422d9680f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:34:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 52319ba5-1182-4ecf-a657-77f6b5096603 kind: Event lastTimestamp: "2026-07-06T21:34:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:34:55Z" name: isvc-autogluon-v2.18bfd0ed46ab17a2 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: fd676fe0-ccbc-472d-ae90-e43d993b5726 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:36:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 52319ba5-1182-4ecf-a657-77f6b5096603 kind: Event lastTimestamp: "2026-07-06T21:36:56Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-06T21:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:36:56Z" name: isvc-autogluon-v2.18bfd109a9084752 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: 4f5f8e76-dc25-453c-8eaa-70c10e60315a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:30:04.336041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17886" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-757779fc66-s4dtm to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a99b54564f namespace: kserve-ci-e2e-test resourceVersion: "17890" uid: 57ebaf8f-b07b-482d-8e1a-3b5cc3599876 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a9a73b6b35 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: 63aa7009-cdf5-4c29-a7e1-f2f216da4c84 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:05Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:30:05Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a9d884a531 namespace: kserve-ci-e2e-test resourceVersion: "17913" uid: f4622d5b-6f31-4f6b-b39e-b80bd9cbc874 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:05Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a9da3c3f3f namespace: kserve-ci-e2e-test resourceVersion: "17916" uid: f09d6f0b-3525-41ea-90b9-328f5e1450c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:05Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a9ddaaefff namespace: kserve-ci-e2e-test resourceVersion: "17917" uid: 63b094e5-020a-4462-8002-e45bd91232df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:05Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0a9de1caef7 namespace: kserve-ci-e2e-test resourceVersion: "17918" uid: 017e209c-45b2-495a-a790-e1e7a13d622a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aad4d7d940 namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: 296a8c67-d1e3-41c9-bc17-95ca7b3451a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aad79309ba namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: 062d9411-2754-4abb-a2ca-a3a7f79fda22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aad8175719 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: e53dccef-c2d3-445b-a777-fbbd93b2d3a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aad821823e namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: c2b7321c-65b5-4587-bd44-93880757884b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aadb2ed69b namespace: kserve-ci-e2e-test resourceVersion: "17959" uid: 53642265-6fcd-424f-9160-32940f380e43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aadba42d32 namespace: kserve-ci-e2e-test resourceVersion: "17961" uid: 6d5f279c-1c6a-4dca-9130-cee10d55b531 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:71a8cdfd0e0680eb03d91cbd024d32c1e390d8fea3237e628b4736412c9b8a37" already present on machine metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aadbafcab6 namespace: kserve-ci-e2e-test resourceVersion: "17962" uid: e25107b1-4f9d-4592-a6b6-778b6ac4011b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aade686957 namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: ae127fae-27ab-4021-b2ae-9d5b35fe12e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:30:09Z" message: Started container agent metadata: creationTimestamp: "2026-07-06T21:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:09Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0aaded97ade namespace: kserve-ci-e2e-test resourceVersion: "17965" uid: ffa806f8-e552-43b0-ad31-e37731589792 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T21:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:41Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0ab10dddb40 namespace: kserve-ci-e2e-test resourceVersion: "18729" uid: f0fa2c59-ff49-4b40-82ab-1dd161404065 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T21:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:41Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0ab10e73948 namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 777d2658-f7d7-416f-8f6b-92bf1c90d291 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:39Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0bfc6ceeee4 namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: e183e9bd-1938-4a75-a533-0bb1cb644440 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:39Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0bfc6d1eb4d namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 68ec44df-d79b-454c-aab3-2cbd90953199 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:39Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0bfc6d39ac1 namespace: kserve-ci-e2e-test resourceVersion: "18674" uid: 0a147c04-6e60-4b30-802c-622a0a5cbb58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-757779fc66-s4dtm namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 74bad355-8e2d-4276-b23a-d713fcca540e kind: Event lastTimestamp: "2026-07-06T21:31:46Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:46Z" name: isvc-logger-predictor-757779fc66-s4dtm.18bfd0c0411e1e98 namespace: kserve-ci-e2e-test resourceVersion: "18761" uid: 03850b02-979b-4dc5-b568-aa0c561984bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-757779fc66 namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: 9bbc31fb-e349-427c-a557-9b8479a89bc9 kind: Event lastTimestamp: "2026-07-06T21:30:04Z" message: 'Created pod: isvc-logger-predictor-757779fc66-s4dtm' metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger-predictor-757779fc66.18bfd0a99a52119a namespace: kserve-ci-e2e-test resourceVersion: "17888" uid: 3abe0292-0a25-4292-90df-68a8d14d9092 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 8d788b53-9fd3-497e-b275-29f199142dd2 kind: Event lastTimestamp: "2026-07-06T21:30:04Z" message: Scaled up replica set isvc-logger-predictor-757779fc66 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger-predictor.18bfd0a992ab23fe namespace: kserve-ci-e2e-test resourceVersion: "17878" uid: f0f232a5-7001-411f-bc2b-a2dad6cb0658 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17874" uid: 99c66599-97f4-4369-b7f0-8ccafc74d2a2 kind: Event lastTimestamp: "2026-07-06T21:30:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:49Z" name: isvc-logger-predictor.18bfd0ad116a0d1c namespace: kserve-ci-e2e-test resourceVersion: "18257" uid: f2bc95e6-f6e0-40e3-86a3-230b7a5669e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17874" uid: 99c66599-97f4-4369-b7f0-8ccafc74d2a2 kind: Event lastTimestamp: "2026-07-06T21:30:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:49Z" name: isvc-logger-predictor.18bfd0ad11c45d4b namespace: kserve-ci-e2e-test resourceVersion: "18258" uid: f94072be-016c-4d81-a007-38f8f05df333 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:31:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: 99c66599-97f4-4369-b7f0-8ccafc74d2a2 kind: Event lastTimestamp: "2026-07-06T21:31:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:31:34Z" name: isvc-logger-predictor.18bfd0b78ddaf39b namespace: kserve-ci-e2e-test resourceVersion: "18585" uid: fda9de60-f40f-42a2-8639-1e1034223ef3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:31:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: 99c66599-97f4-4369-b7f0-8ccafc74d2a2 kind: Event lastTimestamp: "2026-07-06T21:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:31:34Z" name: isvc-logger-predictor.18bfd0b78e414522 namespace: kserve-ci-e2e-test resourceVersion: "18586" uid: cf0a3c86-7e52-4843-83cf-cde1f31de20b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: bfe60e15-773a-495c-8d03-183dc63d992b kind: Event lastTimestamp: "2026-07-06T21:31:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:31:32Z" name: isvc-logger.18bfd0a993146ef8 namespace: kserve-ci-e2e-test resourceVersion: "18568" uid: 8baa6bad-e50b-4d58-b1bd-32977a41c639 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: bfe60e15-773a-495c-8d03-183dc63d992b kind: Event lastTimestamp: "2026-07-06T21:30:04Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger.18bfd0a993b52435 namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: 2dd8187f-b125-4912-8a35-3b145d8803dd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: bfe60e15-773a-495c-8d03-183dc63d992b kind: Event lastTimestamp: "2026-07-06T21:30:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:30:04Z" name: isvc-logger.18bfd0a99fb736fd namespace: kserve-ci-e2e-test resourceVersion: "17897" uid: d55f1f9d-8a87-4525-845c-2cc97f77d2b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "18565" uid: bfe60e15-773a-495c-8d03-183dc63d992b kind: Event lastTimestamp: "2026-07-06T21:31:31Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-06T21:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:31:31Z" name: isvc-logger.18bfd0be0239026e namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: 894bc9bc-fca3-42de-a7b1-ea1fb3ce8ddb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:28:08.977230Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ebf688579 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 45d47619-8b1f-4006-8646-ae918c24e71d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:09Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:28:09Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ed8baf877 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: fbe7c456-72f4-4d12-b617-d1b52180c210 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" already present on machine metadata: creationTimestamp: "2026-07-06T21:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:09Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08eda56d32e namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: b9073973-2a9a-4814-b20f-99af7047b245 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:09Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08edceb8bba namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: 161ebbc0-d9b7-40a0-801d-5c3a2933917f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:09Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08edd6412b9 namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: 1dfcd8ea-7d15-4807-b0e8-a594b2300a5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ff4cc8538 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: e6444891-fe3f-4bb3-b98b-25dd95bb01ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ff796ecd7 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: fcf52433-15f9-49c0-ad0f-7644929e93f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ff80abd27 namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 39dbb182-c258-435d-b867-edfca249c4ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ff8146a11 namespace: kserve-ci-e2e-test resourceVersion: "16977" uid: dbcd8d9d-c541-4bec-964c-883446edca61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ffb1aceee namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: a6bcde64-9dba-439a-8b33-2c54f6e91ded reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ffb8dd56e namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: 711558e1-080b-4496-9745-3509b7a0cf5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:71a8cdfd0e0680eb03d91cbd024d32c1e390d8fea3237e628b4736412c9b8a37" already present on machine metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ffb98688f namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: f3bc7da6-578f-45d1-9862-8a0a9a4d438b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ffe36cd24 namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: e33285e1-85d6-4b3c-b5b1-01a47e7d6cb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:28:14Z" message: Started container agent metadata: creationTimestamp: "2026-07-06T21:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:14Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd08ffeaf0282 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: eab88cb3-8711-4695-b621-fc98a31b02bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T21:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:29:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:52Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd09030c5b860 namespace: kserve-ci-e2e-test resourceVersion: "17768" uid: e8a03ab5-6617-4779-bd2c-6614439a0064 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T21:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:29:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:52Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd0906c95d04c namespace: kserve-ci-e2e-test resourceVersion: "17767" uid: 1ea12fd9-80f3-43a8-befc-e05f6ab4bff0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:44Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd0a4e7555486 namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 3b858df4-6646-455c-bfe4-5973507f0920 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:44Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd0a4e75589df namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: b0a5ae2f-4675-4a9b-a022-d517d4793ac5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:44Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd0a4e755b765 namespace: kserve-ci-e2e-test resourceVersion: "17676" uid: b47b73a7-2028-483a-a9c1-18b3639b5788 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: fe42f0b6-a1a7-42ba-86ba-b1da59c4c2fe kind: Event lastTimestamp: "2026-07-06T21:30:02Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:30:02Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv.18bfd0a59cc75535 namespace: kserve-ci-e2e-test resourceVersion: "17848" uid: d1d621f4-26b5-44fd-b7e3-0f1583bcd258 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-684888dfc6 namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 45dc261c-a608-42da-818e-6f9550e744cd kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-684888dfc6-gwbxv' metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-custom-predictor-684888dfc6.18bfd08ebedce9e6 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 1c26a89a-91a6-44d8-9c05-be297d5796a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: ddc43faf-aefe-4380-9032-125d0f1eb912 kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-684888dfc6 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-custom-predictor.18bfd08eb97dc9a2 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: 2e3c877f-bfcd-4081-8074-d75ab53636d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: bd55ea96-cca4-4b8f-a7a1-1b448f4399c4 kind: Event lastTimestamp: "2026-07-06T21:28:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:28:38Z" name: isvc-sklearn-batcher-custom-predictor.18bfd09238896f47 namespace: kserve-ci-e2e-test resourceVersion: "17168" uid: 7ebb5b1c-28f0-4033-b249-8e0beffda110 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: bd55ea96-cca4-4b8f-a7a1-1b448f4399c4 kind: Event lastTimestamp: "2026-07-06T21:28:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:28:38Z" name: isvc-sklearn-batcher-custom-predictor.18bfd0923927cfda namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: a51c18b2-17dd-4aa1-99ce-5fa73500b076 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: bd55ea96-cca4-4b8f-a7a1-1b448f4399c4 kind: Event lastTimestamp: "2026-07-06T21:29:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:29:38Z" name: isvc-sklearn-batcher-custom-predictor.18bfd099370ea846 namespace: kserve-ci-e2e-test resourceVersion: "17606" uid: ea37094a-4fb7-4964-89ba-e269e024eded reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: bd55ea96-cca4-4b8f-a7a1-1b448f4399c4 kind: Event lastTimestamp: "2026-07-06T21:29:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:29:38Z" name: isvc-sklearn-batcher-custom-predictor.18bfd099377f5213 namespace: kserve-ci-e2e-test resourceVersion: "17607" uid: 561db025-de50-4690-8dfa-831ca47a6b74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: e74f2497-8a94-4bc5-be08-5cb7726cd337 kind: Event lastTimestamp: "2026-07-06T21:29:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:29:32Z" name: isvc-sklearn-batcher-custom.18bfd08eb994dfc6 namespace: kserve-ci-e2e-test resourceVersion: "17560" uid: 9ca124d6-03eb-45f9-8ce8-859be6591cb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: e74f2497-8a94-4bc5-be08-5cb7726cd337 kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-custom.18bfd08eba11fba7 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: ebb80d9d-1c6c-4772-92e3-93ab5544fc98 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: e74f2497-8a94-4bc5-be08-5cb7726cd337 kind: Event lastTimestamp: "2026-07-06T21:29:32Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-06T21:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:29:32Z" name: isvc-sklearn-batcher-custom.18bfd0a226029ba8 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 34cb2b1f-c50e-4dc0-9b82-bc5f7cc6ef83 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:26:13.958020Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-956c77b89-krt5l to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:26:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:26:13Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd073f7bab2dd namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 78ca3a7d-e4f6-48ec-b5a4-289d960bc16b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-batcher-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:14Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07406115243 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: cf0d141c-a022-4cc2-b321-572fb50b5a0a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:14Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:26:14Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07434e457e1 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 2ea34aef-d932-4b99-8448-462da8baa768 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:15Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" metadata: creationTimestamp: "2026-07-06T21:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:15Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd074367c1e13 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 52799ca0-12d6-4b46-997b-369fd7781684 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:18Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:aad9638665b7a403cea616b9df3f859d8e0784845d529e619052a08bd4a49e9e" in 3.699s (3.699s including waiting). Image size: 300568507 bytes.' metadata: creationTimestamp: "2026-07-06T21:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:18Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07512fd651c namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 8e19f4ac-0571-4896-91b8-eeac1bfabbc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T21:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:18Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07516344ca4 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 1d5690b1-e936-4193-86ca-7afc4a71a7a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T21:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:18Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07516a5e9a1 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 7db60a1c-9d16-44e7-92f8-ae4ad16a438a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:22Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-06T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:22Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd0760374e4ab namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 99ad39de-bf6e-44d6-8cee-0d46dbdca199 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 13.631s (13.631s including waiting). Image size: 1558595495 bytes.' metadata: creationTimestamp: "2026-07-06T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:36Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd0792ff0ec2a namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: e6753792-e6f6-4f47-9f68-9acba88ad9a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:36Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07932902506 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 16bb0494-795d-4aa2-a631-fc226379acbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:36Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07932efc172 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 50dcd102-b781-450b-8f51-4146217ae090 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:36Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07932fc1c04 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: fb0b83e5-1273-4463-9cc9-aab5089f0ec9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:38Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.153s (2.153s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T21:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:38Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd079b3591add namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: c49c6348-a000-47eb-a565-cc71912acc4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:38Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd079b751a953 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: babedaaa-4a2a-4464-a004-3fa2297d363a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:38Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd079b7c6aead namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 10909a28-5537-4c02-bc77-82328bad5858 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:38Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:71a8cdfd0e0680eb03d91cbd024d32c1e390d8fea3237e628b4736412c9b8a37" metadata: creationTimestamp: "2026-07-06T21:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:38Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd079b7d4013b namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 76852f90-abb5-4269-8b44-3b8930936a7c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:71a8cdfd0e0680eb03d91cbd024d32c1e390d8fea3237e628b4736412c9b8a37" in 2.268s (2.268s including waiting). Image size: 239997023 bytes.' metadata: creationTimestamp: "2026-07-06T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:40Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07a3f072872 namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 6e60d753-c400-4223-a80e-5e138273de69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:40Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-06T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:40Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07a41acb295 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: bfc534f5-1d04-47a8-b96a-288317e4a678 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:26:40Z" message: Started container agent metadata: creationTimestamp: "2026-07-06T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:26:40Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07a4277a6e8 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 40bf4539-6eb3-4a04-a21b-ede0b5e17b72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T21:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:17Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07a755fa67f namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 462e26e2-731b-4819-b044-6209ccb4a669 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T21:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:17Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd07a7569d6fd namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: 4eeb95d7-12c1-4635-b39c-bf14af0b0d8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd08ebe45c78f namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: e7ad26a1-f1b8-4331-b27f-3451675c06ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd08ebe461884 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: f933c7aa-e8d4-4e5c-8daa-8657d43a9193 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:08Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd08ebe463080 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 34773854-29e5-4faf-a194-15d70c2e4b7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:28:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-956c77b89-krt5l namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 73277ebe-383c-4b73-be93-d5b9639ff3c1 kind: Event lastTimestamp: "2026-07-06T21:28:27Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:28:27Z" name: isvc-sklearn-batcher-predictor-956c77b89-krt5l.18bfd08fa5946554 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 9580c5f2-2461-4a0f-a348-ef10ba5bd5a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-956c77b89 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 1ce60b0e-0fc1-43b7-8492-52c1e853b4ff kind: Event lastTimestamp: "2026-07-06T21:26:13Z" message: 'Created pod: isvc-sklearn-batcher-predictor-956c77b89-krt5l' metadata: creationTimestamp: "2026-07-06T21:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:26:13Z" name: isvc-sklearn-batcher-predictor-956c77b89.18bfd073f72ddf2f namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: c43ef808-41b6-463d-b770-28b5b8cfbf73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 678a38a9-3c93-46b1-be5f-4ae73e55400e kind: Event lastTimestamp: "2026-07-06T21:26:13Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-956c77b89 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:26:13Z" name: isvc-sklearn-batcher-predictor.18bfd073e9381d18 namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: ceab2a7f-5bf8-444b-a041-d6dbe34a2a5d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:26:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 5cfb323e-cdfe-40d4-9df3-2857d588a02d kind: Event lastTimestamp: "2026-07-06T21:27:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:27:13Z" name: isvc-sklearn-batcher-predictor.18bfd0776f0b186c namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: d9e00c0d-6cee-42c1-966d-549589f5eb68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:26:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 5cfb323e-cdfe-40d4-9df3-2857d588a02d kind: Event lastTimestamp: "2026-07-06T21:27:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:27:13Z" name: isvc-sklearn-batcher-predictor.18bfd0776f6e7213 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: ad6433f8-48f6-4149-a6f6-d3dd17b08faa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:26:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 5cfb323e-cdfe-40d4-9df3-2857d588a02d kind: Event lastTimestamp: "2026-07-06T21:27:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:27:58Z" name: isvc-sklearn-batcher-predictor.18bfd07e6d071d2e namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 6b667eb9-4ea7-4403-832d-d1242916ee69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:26:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 5cfb323e-cdfe-40d4-9df3-2857d588a02d kind: Event lastTimestamp: "2026-07-06T21:27:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:27:58Z" name: isvc-sklearn-batcher-predictor.18bfd07e6d6e3798 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 2c1d087e-9246-483e-a9ad-370b65aec3c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T21:26:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 70259892-783f-4db3-a669-86b69d8c5882 kind: Event lastTimestamp: "2026-07-06T21:27:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:27:57Z" name: isvc-sklearn-batcher.18bfd073f014bb93 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: df70d2e2-6559-4a38-b6f4-aeb54ab3852e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:26:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 70259892-783f-4db3-a669-86b69d8c5882 kind: Event lastTimestamp: "2026-07-06T21:26:13Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:26:13Z" name: isvc-sklearn-batcher.18bfd073f126dc4e namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: b06491f1-e2ca-4c2c-baff-861f609c57a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:27:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: 70259892-783f-4db3-a669-86b69d8c5882 kind: Event lastTimestamp: "2026-07-06T21:27:57Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-06T21:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:27:57Z" name: isvc-sklearn-batcher.18bfd08c2e5676e1 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: f12856b5-2900-4153-8abf-9bbe41c67d0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:29:44.239955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-wwslp to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a4ed82aac2 namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: e9403cba-16ef-497d-8291-5c3e0c5b6a0a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dfcf6d9d6-kj6cn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a5070fee3a namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: 0ad4e9e4-2037-454b-9a37-13041515f6ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a5086f2037 namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: 6dee779d-747e-46b9-8858-e1bccdeaa217 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:45Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.242s (1.242s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-06T21:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:45Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a55278debc namespace: kserve-ci-e2e-test resourceVersion: "17714" uid: 67af91f0-bf50-40db-8dae-56bfcc9bf80b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:45Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a5559b9293 namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: c176f8e6-cfb6-425b-9f74-69c12ef685ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:45Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a5562a30ff namespace: kserve-ci-e2e-test resourceVersion: "17716" uid: 02251fe5-d335-487b-a102-199cbc49cc67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:46Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a55633a25a namespace: kserve-ci-e2e-test resourceVersion: "17717" uid: 2988c528-fbcf-4b51-acc3-304737c13d15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:46Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a559b4bcdc namespace: kserve-ci-e2e-test resourceVersion: "17718" uid: 813f8c18-9ea3-4802-9535-a881bf03b270 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:29:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:29:46Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0a55a2b3fff namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: cae47a97-cbd2-4007-9e1b-76c8259cfcd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:39Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0bfc9396dd8 namespace: kserve-ci-e2e-test resourceVersion: "18685" uid: 8928691c-ac08-4d43-8fee-91ec60c246b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-wwslp namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: b3221ede-d447-45e5-acba-8bab4b29b92f kind: Event lastTimestamp: "2026-07-06T21:31:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:31:39Z" name: message-dumper-predictor-6575bcfdfd-wwslp.18bfd0bfc93a852e namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: ad89b3ee-ac09-477a-a3ad-60f3abcd1bba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "17670" uid: 35d0e4e1-766c-4716-bc18-9d7d8f1fc16f kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-wwslp' metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper-predictor-6575bcfdfd.18bfd0a4ece34882 namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 866fdacc-280e-473e-9a99-5635ba2bb598 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: fe6c4668-0022-417b-a830-e44d8bf91634 kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper-predictor.18bfd0a4e83105ae namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: 4ae7374b-68ca-4826-b2b3-b1ca1000a2cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:29:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: 65dd2016-de06-46af-a97b-c7f38a3fa111 kind: Event lastTimestamp: "2026-07-06T21:30:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:14Z" name: message-dumper-predictor.18bfd0a866faabdc namespace: kserve-ci-e2e-test resourceVersion: "18004" uid: e290fa78-6a66-42f8-b190-7f3f7485e22c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:29:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: 65dd2016-de06-46af-a97b-c7f38a3fa111 kind: Event lastTimestamp: "2026-07-06T21:30:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:30:14Z" name: message-dumper-predictor.18bfd0a867639acc namespace: kserve-ci-e2e-test resourceVersion: "18005" uid: 75f2dcb1-dfde-4e62-8c38-161c00314e49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: c8ff41e5-27fd-4a8e-8aa3-d25c800a2d85 kind: Event lastTimestamp: "2026-07-06T21:29:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:29:54Z" name: message-dumper.18bfd0a4e85383e9 namespace: kserve-ci-e2e-test resourceVersion: "17791" uid: 8c2424c2-eae0-4ccd-a672-ffe47f1c0a7f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: c8ff41e5-27fd-4a8e-8aa3-d25c800a2d85 kind: Event lastTimestamp: "2026-07-06T21:29:44Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:29:44Z" name: message-dumper.18bfd0a4e9090887 namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: 3776b1a1-c210-4b00-a5e1-5dae5fb7bf4d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:29:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: c8ff41e5-27fd-4a8e-8aa3-d25c800a2d85 kind: Event lastTimestamp: "2026-07-06T21:29:54Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-06T21:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:29:54Z" name: message-dumper.18bfd0a758512574 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 6e5d4576-6759-43ba-9e56-4c60b4b69e29 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "23527"