--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:24:14.099114Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5876276c7a namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: ffa476f2-8b54-4c80-ba7f-07115a3f5720 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-ts-v1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5881df8c10 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 3d855035-bd79-4219-8e85-dae9b7e4fcd8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:15Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:24:15Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a58b3bd22fe namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 91f3a07d-5bfe-4484-a523-4735f71df9ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:15Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a58b5319e8a namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: ead6ea04-c98e-46d6-896d-f1f07e420431 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:15Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a58b7e9e645 namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: c345f958-5f11-4046-9064-360e40ee52d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:15Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a58b8752639 namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 6ce48655-f4cb-4ae3-af5e-78a281791479 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5adef8f29c namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a36fa9f5-3491-4904-bc94-6d87adb42861 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5ae349e65e namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 6e20a45f-18b2-4814-a270-b6d9b5f165d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5ae3b35280 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 3871c601-b930-4278-9233-40b93a32b46b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5ae3bc2a2e namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 4b8e8e6c-8f6f-403b-b1b9-8f841c56bad0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5ae684bc6b namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 0498941e-be31-4902-bff9-0a92bbb4eb35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:24Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5ae6f525b5 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 9fbea874-6222-4612-aea1-faea44ca00ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:32Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5b56d0df15 namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 34d03147-78c9-42ce-b80b-9ebdf75e0b5f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:24:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:44Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5f752b77f6 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 210d706a-6314-4931-b049-f19a9aedbd65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 328e7fd3-77b8-475a-88d0-d456b0ac7b44 kind: Event lastTimestamp: "2026-07-01T17:24:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:24:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:44Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl.18be3a5f752bce28 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: a3ffe630-f4d9-4818-8e05-5797b1a4a017 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: c08369fa-c419-4616-a60c-0f42ce4e4bd6 kind: Event lastTimestamp: "2026-07-01T17:24:14Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-5fqpl' metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18be3a5875af018c namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: b42f4ec4-4fc9-4078-96b2-fc8fa0bc3d60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 98fdd476-4801-4766-acf4-dc7c3547bb34 kind: Event lastTimestamp: "2026-07-01T17:24:14Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1-predictor.18be3a58710f1e37 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: c33e68f0-fd2e-4935-ac89-e5836f9ad9fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:24:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: 27fa1707-c222-4a85-b358-bb9346a7ef66 kind: Event lastTimestamp: "2026-07-01T17:24:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:24:44Z" name: isvc-autogluon-ts-v1-predictor.18be3a5befeef2b9 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: 3fa9ec1d-995a-4be6-b0e3-2e42a7094891 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:24:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: 27fa1707-c222-4a85-b358-bb9346a7ef66 kind: Event lastTimestamp: "2026-07-01T17:24:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:24:44Z" name: isvc-autogluon-ts-v1-predictor.18be3a5bf0726bec namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: bed23eb5-5279-49dc-9b66-3800badfbfbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: b2e4eb86-eb2d-48e9-805e-6dd76059f8fb kind: Event lastTimestamp: "2026-07-01T17:24:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:42Z" name: isvc-autogluon-ts-v1.18be3a58711c8216 namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 50f67ebe-eace-4d2f-aff1-74e4eaf9a37b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: b2e4eb86-eb2d-48e9-805e-6dd76059f8fb kind: Event lastTimestamp: "2026-07-01T17:24:14Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1.18be3a5871b37754 namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 920e97dc-915d-41f0-8efb-66bed6aa8e47 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: b2e4eb86-eb2d-48e9-805e-6dd76059f8fb kind: Event lastTimestamp: "2026-07-01T17:24:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:14Z" name: isvc-autogluon-ts-v1.18be3a5871b3b9ea namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 891f4ef7-857c-4cea-861e-e8e76158a087 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: b2e4eb86-eb2d-48e9-805e-6dd76059f8fb kind: Event lastTimestamp: "2026-07-01T17:24:42Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-01T17:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:42Z" name: isvc-autogluon-ts-v1.18be3a5f170e0604 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: bd7bb307-9bc7-455f-af4c-45428a61d7e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:14:12.649020Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17239" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:14:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39cc6cef4ad4 namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: 90cb0a29-4bd9-4208-8e33-bc5694d9a9ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:14:13Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:14:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39cc86088f19 namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: fc1d7417-f883-4f2b-b07d-419f84eb9399 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:14:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39cc877b9b76 namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 8c8675e8-d612-4f6b-938d-fe571baec435 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:14:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39cc8a0c5c7f namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 2bd21cc7-ff26-482b-87d1-e2fb18ed0d46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:14:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39cc8aafd321 namespace: kserve-ci-e2e-test resourceVersion: "17282" uid: c6510aa1-db58-4705-9085-dc46f2051dfb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:14:43Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-01T17:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39d38142e5ff namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: ddd57f26-5cdb-4d66-81f7-2bc19507860f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:08Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m25.588s (2m25.588s including waiting). Image size: 8939170586 bytes.' metadata: creationTimestamp: "2026-07-01T17:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f567092d40 namespace: kserve-ci-e2e-test resourceVersion: "18520" uid: 02375073-d5cf-4449-bf36-e9989cb67267 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f5748964aa namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: 3c0070fe-276a-48af-b27d-0816a076dca1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f57597e863 namespace: kserve-ci-e2e-test resourceVersion: "18524" uid: f54b4a6a-6ce4-48a9-9156-fc1bf33b8b87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f575a0f818 namespace: kserve-ci-e2e-test resourceVersion: "18525" uid: 915c0af9-460a-477b-893e-a40a02fe16c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f57b98948a namespace: kserve-ci-e2e-test resourceVersion: "18526" uid: 568ee622-e6aa-421d-9608-80f2129ea676 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:09Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f57cad0956 namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: 3345ca57-5857-449e-98f8-2e4c066692bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:17:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39f5e0808eec namespace: kserve-ci-e2e-test resourceVersion: "18591" uid: 8afcb635-8ed7-4214-98db-94532433b3ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:32Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39fb0eb49c8b namespace: kserve-ci-e2e-test resourceVersion: "18723" uid: da287ca0-bf00-4051-815d-360f65cf453d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: c52be57c-46ef-4e73-9607-611146f0b731 kind: Event lastTimestamp: "2026-07-01T17:17:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:32Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q.18be39fb0eb50d33 namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: a59e484d-f9cd-4e1a-86bd-b5931883db0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 80dfe703-bd6a-4de1-a0d5-b810a82ed3a9 kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-4gb6q' metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:14:12Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18be39cc6c73fec1 namespace: kserve-ci-e2e-test resourceVersion: "17241" uid: af343bbf-b616-4eb6-b148-231e7bcb7752 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 0809092a-0253-4021-85d2-4ff74b4aa1fa kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:14:12Z" name: isvc-autogluon-v1-predictor.18be39cc68f99a03 namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: 59d54202-8eae-45bb-a548-f8580450462c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-01T17:14:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 93d78115-858a-4b24-b2d2-a68f3bc40079 kind: Event lastTimestamp: "2026-07-01T17:17:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:17:27Z" name: isvc-autogluon-v1-predictor.18be39cfe8038eaa namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 81315c80-aa85-4d60-9eaf-172af7605d48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:14:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 93d78115-858a-4b24-b2d2-a68f3bc40079 kind: Event lastTimestamp: "2026-07-01T17:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:17:12Z" name: isvc-autogluon-v1-predictor.18be39cfe86f2c28 namespace: kserve-ci-e2e-test resourceVersion: "18558" uid: 4a6b74a4-1ce4-4602-8ce7-eeb80876f76b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 38c176b7-06d1-4413-8784-238eac16beaa kind: Event lastTimestamp: "2026-07-01T17:17:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:17:26Z" name: isvc-autogluon-v1.18be39cc6924f3f9 namespace: kserve-ci-e2e-test resourceVersion: "18664" uid: 8ab13542-953c-4ef4-930a-75426c98c050 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 38c176b7-06d1-4413-8784-238eac16beaa kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:14:12Z" name: isvc-autogluon-v1.18be39cc69f09077 namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: b6217ed1-12b1-4e0c-a265-defa63f780e7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 38c176b7-06d1-4413-8784-238eac16beaa kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:14:12Z" name: isvc-autogluon-v1.18be39cc69f0ecfa namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: e6c7efad-224c-4f35-9147-72679d15366f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18662" uid: 38c176b7-06d1-4413-8784-238eac16beaa kind: Event lastTimestamp: "2026-07-01T17:17:26Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-01T17:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:17:26Z" name: isvc-autogluon-v1.18be39f9a1987c64 namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: 3ca20f98-c407-454f-9bab-fb7fc5abd638 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:21:58.794257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:21:58Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a38f55b2d7c namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: cac0bb8a-2459-4473-87d6-9fb009b6d351 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:21:59Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:21:59Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a390f1e1f6e namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 50e221b6-06bf-47fd-a2d3-46d381f4bf85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:21:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:59Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a39106fa5ce namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: e6146f5b-c3ed-4e83-aa5d-027810fbaab5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:21:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:59Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a3913014912 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: d6ef3fa4-73c6-4b03-ae7e-9e7ad3dcea7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:21:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:59Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a39136ef4f9 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 224248be-66e0-408a-a7b6-344fbca01375 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:28Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T17:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:28Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a3ffbf8ec49 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 9df640c6-742d-487e-8890-43dad403a717 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:29Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a3fff135a19 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: de48dd01-2d3d-4fa1-88ed-b7f36e285c82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:29Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a3fffa1230d namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 12e7c501-1529-4147-b64b-0f99364ff260 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:29Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a3fffaa1237 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: c328a1a4-30c0-48f5-b484-de254f2f4a4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:29Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a4002ac1af0 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 9bd3b966-fc25-49db-a94a-0dc29e034375 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:22:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:22:29Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a40031b28f0 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 62135474-c515-48e4-8840-6165a5ae939b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:24:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:24:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:08Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a574457653d namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 0139c66d-4a2f-4dd7-89bf-f4b59078e877 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 4fbc0146-d07c-4375-b326-13db3405ecbb kind: Event lastTimestamp: "2026-07-01T17:24:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:24:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:24:08Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj.18be3a574457c10d namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 15366dcc-61bc-4233-96df-92edb13a0025 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: e5212666-adb5-4465-af4f-d7bcdd5f9ea8 kind: Event lastTimestamp: "2026-07-01T17:21:58Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-fcdzj' metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:58Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18be3a38f4f059d3 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: e97c7dec-2bea-4e41-ab69-ecb7f407ee46 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: a7c42067-3412-4e17-a5cb-01d993942a9f kind: Event lastTimestamp: "2026-07-01T17:21:58Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:58Z" name: isvc-autogluon-v2-noslash-predictor.18be3a38f034b9e6 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 25485af9-108d-4b3a-b6fc-1c016b0d9613 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 28be464d-4f45-4d12-8f5c-6c892a4d3414 kind: Event lastTimestamp: "2026-07-01T17:22:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:43Z" name: isvc-autogluon-v2-noslash-predictor.18be3a3c6f377409 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 76f06b98-4276-47c3-92de-e7fb22bef7f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 28be464d-4f45-4d12-8f5c-6c892a4d3414 kind: Event lastTimestamp: "2026-07-01T17:22:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:43Z" name: isvc-autogluon-v2-noslash-predictor.18be3a3c6fa37ef5 namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: fa36c9ce-33f8-4f4b-b1db-ea11c096d4be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:22:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 28be464d-4f45-4d12-8f5c-6c892a4d3414 kind: Event lastTimestamp: "2026-07-01T17:23:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:23:58Z" name: isvc-autogluon-v2-noslash-predictor.18be3a46ed7b92b1 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: a6a741e4-46cf-4669-99d3-5f48ba5e3b98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:22:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 28be464d-4f45-4d12-8f5c-6c892a4d3414 kind: Event lastTimestamp: "2026-07-01T17:23:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:23:58Z" name: isvc-autogluon-v2-noslash-predictor.18be3a46edec8c62 namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 6709b743-b417-4915-88fb-fee2f9de4a97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: e76cd8a9-4518-4e81-8ede-c64cc8f7f828 kind: Event lastTimestamp: "2026-07-01T17:24:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:05Z" name: isvc-autogluon-v2-noslash.18be3a38f0b6119f namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 60404d24-75f5-4ae9-ae82-8b44603442f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: e76cd8a9-4518-4e81-8ede-c64cc8f7f828 kind: Event lastTimestamp: "2026-07-01T17:21:58Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:58Z" name: isvc-autogluon-v2-noslash.18be3a38f12121da namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: d52d444c-bad6-4231-a4ed-526470dfcafb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: e76cd8a9-4518-4e81-8ede-c64cc8f7f828 kind: Event lastTimestamp: "2026-07-01T17:21:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:58Z" name: isvc-autogluon-v2-noslash.18be3a38f1217a0e namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 08b814f3-ecc6-46e6-addf-367fad74627a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:24:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: e76cd8a9-4518-4e81-8ede-c64cc8f7f828 kind: Event lastTimestamp: "2026-07-01T17:24:05Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-01T17:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:24:05Z" name: isvc-autogluon-v2-noslash.18be3a566c302d84 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: 715623b9-8523-48ce-aba2-8e08f17a26e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:17:38.229131Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-bmr8g to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc4a76dc7d namespace: kserve-ci-e2e-test resourceVersion: "18790" uid: 41ca6497-e225-4ed5-8946-08ba79897aaa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:17:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc5acc2a5b namespace: kserve-ci-e2e-test resourceVersion: "18801" uid: f601f505-8772-4ddd-9738-dbc50d54f9a7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:17:39Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:17:39Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc8b579e7f namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 24f30be6-e02d-4b65-8e77-723c01ad3e82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:17:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:39Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc8d1eeb5e namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: a2dd107d-78d6-4c6d-b32a-ee75a926baf5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:17:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:39Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc9288d91b namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: b1cba014-68b3-4aaa-9375-5e82e7d74292 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:17:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:17:39Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be39fc939ad986 namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: cd51dfa6-82d5-4da2-a847-72d8b5849a1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:10Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T17:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:10Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03e6f0a20e namespace: kserve-ci-e2e-test resourceVersion: "19044" uid: 2984862a-ba3c-4e4c-84d4-3d882ca78b6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:10Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03ea00388c namespace: kserve-ci-e2e-test resourceVersion: "19046" uid: 93bde88c-15a1-412b-ad77-7ad79b446d84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:10Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03ea68b48b namespace: kserve-ci-e2e-test resourceVersion: "19047" uid: c0236a52-908c-48fd-80f8-ea16b3591d17 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:10Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03ea72b85f namespace: kserve-ci-e2e-test resourceVersion: "19049" uid: 27d48e5e-6ec3-4071-9d89-fbf15860fc23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:11Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03ecd4095f namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: 51ba64df-7d92-42d7-9c0b-09304019b3a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:18:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:18:11Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a03ed30dc75 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: 587cf18f-1c72-4d0e-8f80-651d90a0e6da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:19:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:48Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a1a99193e17 namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 77bdbc62-02a1-4912-b414-fbe050f84dfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:19:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:48Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a1a9919a12d namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: baa54bee-721c-46c2-b778-54221679fa2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-bmr8g namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: b242f7cd-ff9f-4f8c-a9df-2b6951290757 kind: Event lastTimestamp: "2026-07-01T17:19:48Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:48Z" name: isvc-autogluon-v2-predictor-67b849d456-bmr8g.18be3a1ab9041ef4 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 279d3999-7d08-4b70-8fd3-c32aafb3104e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: 56fe26b7-cdbb-4c60-b3bb-a634136e3f1d kind: Event lastTimestamp: "2026-07-01T17:17:38Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-bmr8g' metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2-predictor-67b849d456.18be39fc4a048135 namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: af797d12-e0fc-4bff-9bb8-8ce4bbcfecd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: db09c22f-d6ec-4aa3-b30f-034274c6ac69 kind: Event lastTimestamp: "2026-07-01T17:17:38Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2-predictor.18be39fc44d53c19 namespace: kserve-ci-e2e-test resourceVersion: "18779" uid: 388baa1d-31ac-44c2-b186-a0d283e3f6fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 6a8c2b62-53b6-49fa-9970-ebaf35150484 kind: Event lastTimestamp: "2026-07-01T17:18:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:18:38Z" name: isvc-autogluon-v2-predictor.18be39ffc39ccf28 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: 6c2f13ce-d578-40f1-933a-f277caacc160 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 6a8c2b62-53b6-49fa-9970-ebaf35150484 kind: Event lastTimestamp: "2026-07-01T17:18:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:18:38Z" name: isvc-autogluon-v2-predictor.18be39ffc403f1bf namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: e4afff8f-102d-407a-aef6-50a4aac7a8bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18915" uid: 6a8c2b62-53b6-49fa-9970-ebaf35150484 kind: Event lastTimestamp: "2026-07-01T17:19:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:38Z" name: isvc-autogluon-v2-predictor.18be3a0dc084d78b namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: f5e45835-fc4d-4a8c-a682-c751c6e4a24d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18915" uid: 6a8c2b62-53b6-49fa-9970-ebaf35150484 kind: Event lastTimestamp: "2026-07-01T17:19:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:38Z" name: isvc-autogluon-v2-predictor.18be3a0dc12194ce namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: f6aa7917-ed77-4b9b-956d-c20140b4b034 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:19:53.497247Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1bc9128e4a namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: c2728015-9f2b-4aa3-9cea-e594a84f42bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1be24bb9be namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 5b549062-3965-491a-8e8e-bc8385167785 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1be3e13c76 namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 306b4281-d8b4-4f28-9442-285507b9be06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1be6a13af1 namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 8c595784-41e4-4893-a01f-385bea639300 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:19:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:54Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1be72940ab namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 14c27189-5696-4df9-a34c-26e07d1e7741 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ff4bf62d4 namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: b46ed060-c8d2-4bbb-badb-283e8f471995 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ff7a81a68 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 2c3a01e0-bbd7-4baa-8754-37172085865e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ff81a0ddd namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: eabd9f29-75bb-413d-bcb4-2a218f0b857f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ff8249759 namespace: kserve-ci-e2e-test resourceVersion: "19966" uid: fe297f53-ee27-45a3-989c-5fab50e8da01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ffaefe895 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 0eda45b3-f89c-4d82-b72e-3f3dbf0f5bba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:20:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:11Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a1ffb62c066 namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: af231bd4-9edc-4f85-8153-75eb4c663230 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:21:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:21:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a37c24c0d60 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 86a50df9-0907-4c63-a3ab-b298e224c845 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:21:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:21:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a37c24ca814 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 8fcbfe50-cc40-415a-b365-d46efbd6c723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 9e61a7b3-fb58-49db-a04d-bbbfcde73fe4 kind: Event lastTimestamp: "2026-07-01T17:21:54Z" message: 'Readiness probe failed: Get "http://10.133.0.31:8080/v2/models/isvc-autogluon-v2-variants/ready": dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:21:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:54Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j.18be3a37e6320a99 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 7cc7137f-f5d1-4358-b44a-320a427ec1f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 9b6921f9-0402-4445-b376-7b5e7d748da4 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-ff48j' metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18be3a1bc8a51165 namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: cad6f6f1-185a-43f4-90a2-52d32bc6f604 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 7886bfc6-7d89-486a-9ab9-86dd4c85b983 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants-predictor.18be3a1bc3ae7ac4 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: 0eff3d90-1050-4c31-8f0e-a0f8052300d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: ba9ea316-95ae-4994-9f5f-79a34e699024 kind: Event lastTimestamp: "2026-07-01T17:20:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:38Z" name: isvc-autogluon-v2-variants-predictor.18be3a1f42e7961d namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: b39e6431-a041-4e09-b25c-abca6768854f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: ba9ea316-95ae-4994-9f5f-79a34e699024 kind: Event lastTimestamp: "2026-07-01T17:20:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:38Z" name: isvc-autogluon-v2-variants-predictor.18be3a1f43578f6d namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: c70dd941-da54-4821-8b50-b7667d74f991 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:20:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: ba9ea316-95ae-4994-9f5f-79a34e699024 kind: Event lastTimestamp: "2026-07-01T17:21:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:53Z" name: isvc-autogluon-v2-variants-predictor.18be3a29c090dbed namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 000ccaf8-3c12-4987-a65a-6f94ec622241 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:20:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: ba9ea316-95ae-4994-9f5f-79a34e699024 kind: Event lastTimestamp: "2026-07-01T17:21:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:53Z" name: isvc-autogluon-v2-variants-predictor.18be3a29c10717f9 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: e675e76d-d3f7-4d29-a7f9-8b0c149283e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 4b6c6027-1e9e-448a-b89f-99023c3a1b39 kind: Event lastTimestamp: "2026-07-01T17:21:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:44Z" name: isvc-autogluon-v2-variants.18be3a1bc3cdd79b namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 868e090b-bee6-4e0f-a0fa-e49e4712510b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 4b6c6027-1e9e-448a-b89f-99023c3a1b39 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants.18be3a1bc445c25e namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 774f54ce-2e4c-4e50-93ca-6fade29acb0f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 4b6c6027-1e9e-448a-b89f-99023c3a1b39 kind: Event lastTimestamp: "2026-07-01T17:19:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:53Z" name: isvc-autogluon-v2-variants.18be3a1bc445fd92 namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: f8730065-d439-476b-80f7-d3278329c3aa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 4b6c6027-1e9e-448a-b89f-99023c3a1b39 kind: Event lastTimestamp: "2026-07-01T17:21:44Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-01T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:44Z" name: isvc-autogluon-v2-variants.18be3a3598036241 namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 58c27ffb-e106-4c14-a7d3-32f2b34d78f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18768" uid: ac53993f-9762-412c-bf1d-7615a7109445 kind: Event lastTimestamp: "2026-07-01T17:19:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:42Z" name: isvc-autogluon-v2.18be39fc44d87c17 namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 7a1f9f4b-67cd-47d2-832f-4901c0a0f748 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18768" uid: ac53993f-9762-412c-bf1d-7615a7109445 kind: Event lastTimestamp: "2026-07-01T17:17:38Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2.18be39fc452fae2d namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 90df70c0-e345-4ce1-b921-e46cd0dde4ba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:17:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18768" uid: ac53993f-9762-412c-bf1d-7615a7109445 kind: Event lastTimestamp: "2026-07-01T17:17:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:17:38Z" name: isvc-autogluon-v2.18be39fc452fefec namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 07002444-fe8c-4a48-8af1-0e0c40be776b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: ac53993f-9762-412c-bf1d-7615a7109445 kind: Event lastTimestamp: "2026-07-01T17:19:42Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-01T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:42Z" name: isvc-autogluon-v2.18be3a193035a0b4 namespace: kserve-ci-e2e-test resourceVersion: "19681" uid: 8c52bcd3-7994-4690-bd06-1ecc7386f5cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:12:47.426717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-599b89d7c-p5d9m to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b8954a2328 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 3acb9adf-1f98-4958-bba0-eae6a3e4136a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b8ae60d5d9 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: cc4406d3-b4f8-4aa3-933a-68b9b9bbb416 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b8b01134ca namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: b7beae71-42fd-42dd-9892-0cf2f5c9fa1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b8b2a9165b namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 2af84e99-f881-4d14-9dee-86040f88cc29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b8b314a069 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: cea3e995-7b56-4504-b382-15aa302e3e96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b98be6fe29 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 12a34b49-61d8-481b-8196-7c8a44bee856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b98f1d34d9 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: d4253095-45cc-4e6b-b08b-5c0bbb54e46d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b98f8ece2a namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: d7995a0a-880f-4b6b-923a-e6f5d870a22c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b98f984b4b namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 625c7386-1d50-4caa-9c30-7d348baab317 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b9921ef5ca namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: d51c98e1-9f9e-44cb-a1fe-d1ae9f40ff95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b9928f4870 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: a82fe5f9-aede-4921-b418-163a15ea27e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:6e0ddcf7f1337e58596baa7c9b81109d1b3a245f58dd3bd46131f522391a7237" already present on machine metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b99298a8bf namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: d253492b-bba2-4508-9402-b7f8c1770266 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b9958f84ae namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 2837b546-dd8e-4cbd-8de6-11767ec3e7af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:12:51Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T17:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:51Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b995ff2e68 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: ab19209c-28c5-4c24-a67a-d87cb2590de8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:25Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39b9c7ebd59d namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 0661f621-2e14-4202-ad0e-bb106b946d46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T17:12:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:12:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:15Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39ba03d0c0dc namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 0a0992a9-4d3d-40d6-a95f-ab649d37f587 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:12Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39cc6b71f924 namespace: kserve-ci-e2e-test resourceVersion: "17236" uid: 87374a76-4892-4354-ae99-1aa368875f6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:12Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39cc6b722b6b namespace: kserve-ci-e2e-test resourceVersion: "17238" uid: 1d5de2cb-b38c-4a71-81ca-35e7c1a913eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:12Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39cc6b726a3f namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: bb7ba8de-6b1b-40ab-9d3a-e395998c104d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-p5d9m namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: ae0049f6-86e1-43c6-b675-21b3e853802c kind: Event lastTimestamp: "2026-07-01T17:14:24Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:24Z" name: isvc-logger-predictor-599b89d7c-p5d9m.18be39ccdff8712e namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: 27af07f7-bb01-4bd2-9061-01dbc8449cf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-599b89d7c namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 8db1bca5-a8f3-43ef-bcd9-9c04f84c8232 kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: 'Created pod: isvc-logger-predictor-599b89d7c-p5d9m' metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor-599b89d7c.18be39b894c55ed1 namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: f09097ea-77f5-4ee3-bf72-3aa820455655 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 79261630-1ba1-4bd3-ab59-bd82f12904ce kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: Scaled up replica set isvc-logger-predictor-599b89d7c from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger-predictor.18be39b890d58b7f namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 5a668743-74c7-42fe-8a34-ce67f5d0ff2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:13:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: f21aafd6-7718-4434-9282-d1bec698626b kind: Event lastTimestamp: "2026-07-01T17:13:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:13:32Z" name: isvc-logger-predictor.18be39bc0f55f638 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 78af9c34-a413-4071-a8bc-a21c82597ce7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:13:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: f21aafd6-7718-4434-9282-d1bec698626b kind: Event lastTimestamp: "2026-07-01T17:13:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:13:32Z" name: isvc-logger-predictor.18be39bc0fbba295 namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 9ccfae9e-27d3-4efa-88cc-64271dff53f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:13:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: f21aafd6-7718-4434-9282-d1bec698626b kind: Event lastTimestamp: "2026-07-01T17:14:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:14:02Z" name: isvc-logger-predictor.18be39c68bf87028 namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 2d8f3c5a-3643-47b5-9f38-8c5a5cc364d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:13:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: f21aafd6-7718-4434-9282-d1bec698626b kind: Event lastTimestamp: "2026-07-01T17:14:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:14:02Z" name: isvc-logger-predictor.18be39c68c6456e0 namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: 2be8f686-d113-46a8-82cd-e434e637fa69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 6d4711ec-cd56-4ac5-95ff-5a5c2d1779e0 kind: Event lastTimestamp: "2026-07-01T17:14:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:14:05Z" name: isvc-logger.18be39b8910b8de7 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: b074f57b-8ec5-4bcf-bddb-88aaf93c0f66 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 6d4711ec-cd56-4ac5-95ff-5a5c2d1779e0 kind: Event lastTimestamp: "2026-07-01T17:12:47Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:47Z" name: isvc-logger.18be39b8916ab8b6 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: e08b4713-bfd5-4129-8636-963e3deae991 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 6d4711ec-cd56-4ac5-95ff-5a5c2d1779e0 kind: Event lastTimestamp: "2026-07-01T17:14:05Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-01T17:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:14:05Z" name: isvc-logger.18be39cabb353d91 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 6fdbfb33-8da3-4e1a-af58-fd66a402dbc9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:11:02.235806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a0176c2aad namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: cbd934bb-2359-4688-8b31-af856512f189 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a030f0e438 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: e33b100e-da6b-4d55-860b-57989702df35 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a032c10a55 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: a5193fea-90e3-4047-a704-85ade7f186e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a03588f58f namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 2d52cca6-e4a2-40a4-8f1a-142097785ccd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a035f71719 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 42f7ca19-eb77-401b-8a19-9da60666fe08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a13a29da29 namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 0ddbcc53-2a13-4625-a597-f79eb022a737 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a13dd25174 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 065dc10c-f911-4113-bced-3f1502f6c5f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a13e442766 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: b1d1df5e-f989-44ac-827f-132cc0c3f1c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a13e4e6bcc namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 106a41f0-4144-4daa-835d-2f8621296009 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a140fffe08 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e7ec93f0-c21f-4f47-9323-de473be891da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a1416f363e namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 3004f679-c014-45e1-94f4-97d65ddfcf14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:6e0ddcf7f1337e58596baa7c9b81109d1b3a245f58dd3bd46131f522391a7237" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a141790c9c namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 883208c5-17e5-4459-b58c-4a2a76264b6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a143febd42 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: 41fa237d-3c89-479b-96df-a92c9d45129d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T17:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a1446a8b9e namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: b5570d25-8729-4a1f-8ad5-9d294ac231a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:11:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:11:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:36Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a17690bb2f namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: cd7c7425-2200-4b42-9945-32c10e6023e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T17:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:36Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39a1b26db97c namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 7bc5fc5a-d9b4-454e-8e41-88e4205524ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:27Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39b3e59ae29c namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 5a945bb8-2f51-468d-8e77-79bdf3803b81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:27Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39b3e59b1b93 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 8d3fa874-6047-48ff-8111-d6325386177a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:27Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39b3e59b7c2a namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 391488a5-7c69-43c4-bd15-812040c38333 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:12:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f7c7e394-4643-49ef-b2ca-dff32f41ff0b kind: Event lastTimestamp: "2026-07-01T17:12:45Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:45Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st.18be39b48ea549f7 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 5aec5174-de72-4d68-9948-a532e626161c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 232989fc-f923-48f6-9c6b-4ead9c7a339a kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-vc7st' metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5.18be39a016bb7354 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 59dfa865-36e4-4c5b-b1a1-a7e3f81db1dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: bbff09d4-79af-46c1-83c5-dd8893738bd7 kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-5fccccdbd5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom-predictor.18be39a010eb7dd9 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: ad07f3a8-fc30-4163-b29e-7bb7b7f58637 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: f75419db-cf8d-49a3-9204-05209e9473cd kind: Event lastTimestamp: "2026-07-01T17:11:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:47Z" name: isvc-sklearn-batcher-custom-predictor.18be39a38f7c9b66 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 573d4e90-f19c-4108-abbc-d6554155837e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: f75419db-cf8d-49a3-9204-05209e9473cd kind: Event lastTimestamp: "2026-07-01T17:11:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:47Z" name: isvc-sklearn-batcher-custom-predictor.18be39a39010a969 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: cecdb891-5b57-45c4-9c3f-50b91c5f62c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:12:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: f75419db-cf8d-49a3-9204-05209e9473cd kind: Event lastTimestamp: "2026-07-01T17:12:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:17Z" name: isvc-sklearn-batcher-custom-predictor.18be39ae0d6c618b namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 92621a9c-9993-4a69-b3fa-2a7e22c63015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:12:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: f75419db-cf8d-49a3-9204-05209e9473cd kind: Event lastTimestamp: "2026-07-01T17:12:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:17Z" name: isvc-sklearn-batcher-custom-predictor.18be39ae0de1eac9 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 1c460b0e-7741-472f-bdf2-a2a9049e547c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: d6b13002-14bf-46da-81c5-c0a331e681db kind: Event lastTimestamp: "2026-07-01T17:12:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:16Z" name: isvc-sklearn-batcher-custom.18be39a01122fc8c namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: e082957d-1cf1-48d8-8e46-fb51b5702ec5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: d6b13002-14bf-46da-81c5-c0a331e681db kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-custom.18be39a0117d32d5 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 52c7f4cb-5e43-463b-97d4-0cc30c232a82 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: d6b13002-14bf-46da-81c5-c0a331e681db kind: Event lastTimestamp: "2026-07-01T17:12:16Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-01T17:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:16Z" name: isvc-sklearn-batcher-custom.18be39b15969c113 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: fffe9f26-e1f9-4318-bc6e-0df67623356e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:09:07.059059Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be3985465a90f7 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 6c87d376-81df-4616-ae73-f52cabdd9f85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:07Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39855fa891bf namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 0f945155-6b2f-4fa1-80d8-78a8cebbbb1e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:07Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be3985615e1d14 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 85aa2f13-34b2-4697-a96f-9b7be0bd2396 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" in 3.883s (3.883s including waiting). Image size: 301878744 bytes.' metadata: creationTimestamp: "2026-07-01T17:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:11Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398648dd58d8 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 7446b400-5ecf-4b83-b37c-a5e04c89dd71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:11Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39864bc8f2e8 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 2c7d5bef-409b-48ed-ab30-42dfbeee95ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:11Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39864c398679 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 5d9b7bb7-5f09-40e6-8913-b5250718a8ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T17:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:15Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398745f5c6fd namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: f816edcf-9136-46df-81ae-a0164dc560de reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:28Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 13.066s (13.066s including waiting). Image size: 1555871157 bytes.' metadata: creationTimestamp: "2026-07-01T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:28Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398a50c67862 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 481f6809-6704-49b7-baa6-c1ec5365bdaa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:28Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398a540c954c namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: e94f509c-d3d4-4dcf-8857-6e74f86d2dc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:28Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398a54779555 namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: d4f24ce6-eedd-41b3-8113-bccc584fa6c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:28Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398a54843f1f namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 3b3eb95b-9a0b-40ea-a1f6-cae120fc9458 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.269s (2.269s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:31Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398adbcf1719 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 5a46c6d8-d206-48ba-b4cb-b6369674f2d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:31Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398ade7cab6e namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 335d8b7e-4fb1-4e16-891c-b610d5d3dbda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:31Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398adee8610c namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 83bebe53-2678-42ee-9c3d-7db9a8d350ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:31Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:6e0ddcf7f1337e58596baa7c9b81109d1b3a245f58dd3bd46131f522391a7237" metadata: creationTimestamp: "2026-07-01T17:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:31Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398adef3bd9f namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 319255cf-5c4b-4d12-a588-0affa91c253f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:33Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:6e0ddcf7f1337e58596baa7c9b81109d1b3a245f58dd3bd46131f522391a7237" in 2.685s (2.685s including waiting). Image size: 239992444 bytes.' metadata: creationTimestamp: "2026-07-01T17:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:33Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398b7f088f4f namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 367fe4b9-926f-4f3b-91f1-12ba18f2b69e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:33Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T17:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:33Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398b81d289d9 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 5678bdd9-326f-43bd-9ae0-8e8dba665d43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:09:33Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T17:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:33Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398b826a6f15 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 189b478e-74cf-4edc-bfd7-73d0ebbb217b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:09:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:07Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398bb8082352 namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 83b8efea-1a24-4a6d-9bf4-ca1a12a35d65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:17Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be398bf3e6f991 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: bb08b3fa-9449-41d3-951a-d04d74ead1b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39a016883fa2 namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 552b47e8-ee35-4bb6-94fa-7c84baf342be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39a016888e7b namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: a3c46cfa-c65e-4a2d-97c5-5b0f966534a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:02Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39a01688bcc2 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: c05eb63a-ffcb-437a-8b62-659a865606e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 98aa0b13-a938-471c-92d2-e0a9e84651b8 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:16Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq.18be39a1241e0543 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 51ac27bb-3ad2-4a50-85ad-b049b610f54a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-869f8b8c7d namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 67d1489c-c45c-44cc-a05a-adb99eeed1f8 kind: Event lastTimestamp: "2026-07-01T17:09:07Z" message: 'Created pod: isvc-sklearn-batcher-predictor-869f8b8c7d-mn6kq' metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d.18be398545eab36f namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: ce612eb5-3cbb-4dff-9609-726eb5700164 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 11b6b6ee-22b9-48de-907a-4246fff3862c kind: Event lastTimestamp: "2026-07-01T17:09:06Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-869f8b8c7d from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:09:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:06Z" name: isvc-sklearn-batcher-predictor.18be39853e0f5ec8 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: e8bcb8f3-198b-4d5c-818c-e2e8f6a13c2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: d011bdee-7d10-4eef-a322-0edf8baabe3c kind: Event lastTimestamp: "2026-07-01T17:10:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:07Z" name: isvc-sklearn-batcher-predictor.18be3988c34c64cc namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: 7883a618-fab4-42fe-91a2-15bff9e89a49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: d011bdee-7d10-4eef-a322-0edf8baabe3c kind: Event lastTimestamp: "2026-07-01T17:10:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:07Z" name: isvc-sklearn-batcher-predictor.18be3988c3b450f2 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: e9f071f2-6365-4919-9586-ab63b10661f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:09:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: d011bdee-7d10-4eef-a322-0edf8baabe3c kind: Event lastTimestamp: "2026-07-01T17:10:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:09:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:52Z" name: isvc-sklearn-batcher-predictor.18be398fc1b7ef1c namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: edbeee12-ba07-4f42-8add-f063ec07d65f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:09:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: d011bdee-7d10-4eef-a322-0edf8baabe3c kind: Event lastTimestamp: "2026-07-01T17:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:09:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:52Z" name: isvc-sklearn-batcher-predictor.18be398fc22abc9c namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: c9402b1b-dd94-4d35-bac5-617c95dddcc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 185be31d-415c-4af2-b542-6369db619c0b kind: Event lastTimestamp: "2026-07-01T17:10:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:47Z" name: isvc-sklearn-batcher.18be3985442d2a2f namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 9a7cfe99-5738-4dc0-b922-1f3863d8d6f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 185be31d-415c-4af2-b542-6369db619c0b kind: Event lastTimestamp: "2026-07-01T17:09:07Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher.18be398544879321 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: de82a52f-5c22-4bf8-8f5c-c2d1cfa8c991 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:09:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 185be31d-415c-4af2-b542-6369db619c0b kind: Event lastTimestamp: "2026-07-01T17:09:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:07Z" name: isvc-sklearn-batcher.18be39854487d2fd namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 5116ae63-ed37-4cc6-9e7c-88908ae7b474 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 185be31d-415c-4af2-b542-6369db619c0b kind: Event lastTimestamp: "2026-07-01T17:10:47Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-01T17:10:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:47Z" name: isvc-sklearn-batcher.18be399cc3160c06 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 9085826a-c6aa-446e-81a0-2a1c9a9be30a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:12:27.342493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-mn29k to ip-10-0-137-244.ec2.internal metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:12:27Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b3e82d32d4 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: df0c82eb-8f3f-48de-ba6a-4153f22ab725 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb58c5f57-7bscn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:27Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b3f70be88a namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: e0bd76d7-e8d8-4bfe-8614-ec8635d1d101 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:28Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:28Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:12:28Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b42527e202 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 16037aa6-effd-4bf1-ae49-a323dd4032ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:28Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-01T17:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:28Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b426c5ae9b namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 2f437ed3-41d3-4e31-99f6-d85bcdef4594 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.235s (1.235s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b470717586 namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: e678b777-1a54-4768-835a-06bfe82c1e89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b4733d1886 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: d9074eb7-cab5-45bb-a17c-ee0a313681c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b473aaee3d namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 4ac25d1f-8a84-4c72-b958-fe871f466e14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b473b3df9c namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 629ad9f9-e348-42fb-823d-17df86bd6adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b47639961a namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: bdf17dd6-96a8-4ff5-8f59-9cdb5ed005d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:12:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:12:29Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39b476a47d67 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 31541c2d-061a-4182-93e8-4d373fa81429 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:12Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39cc715358d9 namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: e952d583-2b97-4020-8c15-a628007600ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mn29k namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3c3bda14-25ec-4abd-b396-4e783e462da4 kind: Event lastTimestamp: "2026-07-01T17:14:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:14:12Z" name: message-dumper-predictor-6575bcfdfd-mn29k.18be39cc7153c94f namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: a02828fe-1e82-409f-b995-3d2b5a8e5015 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-244.ec2.internal source: component: kubelet host: ip-10-0-137-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: c8efc015-d8a3-43ba-8fe8-e1b1e7d7610b kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-mn29k' metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:27Z" name: message-dumper-predictor-6575bcfdfd.18be39b3e779020c namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 427205b8-a020-47db-a9d3-a5252b6fc973 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 854d0dbe-3870-4f88-8328-590f15dee795 kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:27Z" name: message-dumper-predictor.18be39b3e544c6aa namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: df14e4ed-3fe2-45a9-9377-86a59ae2d6d5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:12:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 12f53069-7a1c-4643-a26e-32e2710f918b kind: Event lastTimestamp: "2026-07-01T17:12:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:57Z" name: message-dumper-predictor.18be39b7638f1631 namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: 3821af56-019a-4044-abb0-80945466c705 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:12:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 12f53069-7a1c-4643-a26e-32e2710f918b kind: Event lastTimestamp: "2026-07-01T17:12:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:12:57Z" name: message-dumper-predictor.18be39b763faba41 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 9651a369-03ad-4f26-a400-f737b8b71e54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: b01f2a47-d04d-4b19-8164-cb110fc3f131 kind: Event lastTimestamp: "2026-07-01T17:12:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:37Z" name: message-dumper.18be39b3e500dd62 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: deb65b50-e16a-468c-bd11-0df6be196196 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: b01f2a47-d04d-4b19-8164-cb110fc3f131 kind: Event lastTimestamp: "2026-07-01T17:12:27Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:27Z" name: message-dumper.18be39b3e56fe943 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 8822d022-87ed-41e5-9efb-6cfb29bb0a43 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:12:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: b01f2a47-d04d-4b19-8164-cb110fc3f131 kind: Event lastTimestamp: "2026-07-01T17:12:37Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-01T17:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:12:37Z" name: message-dumper.18be39b649c27bac namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 534ba5ed-8494-45f0-8844-95b4e3f85750 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22306"