--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:58:27.823984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:58:27Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9a664fa26 namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 10363609-c1cc-40af-b28c-b563596c6deb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-ts-v1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:28Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9b4962bec namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: f478e811-abbc-456f-abd6-af41faba079d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:28Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:58:28Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9e36285dd namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 6d9bc8b0-a3e6-4eb4-99d4-b3b10da511c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:28Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9e4fbc14d namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 184e9e57-8348-462f-a7aa-3e2c360eee0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:28Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9e7758974 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: f78b0af6-3a95-44a1-9b73-b391b4aac43b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:28Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096a9e7e1bd58 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 8ce18b32-d97a-46d2-a14b-dbbfd77c40be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac14707e3c namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: aa1fe60a-976a-4774-b9f2-af563063aba6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac17873e0e namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: f2c5ea83-001d-4686-a078-a08ea3b34492 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac17f45192 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: fd552479-6a54-4b91-b6ed-cfba355f851d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac17fee849 namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: c03054ed-b69e-452d-bce4-158f35a3fee2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac1b5febb6 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: b13d6f12-653f-4513-98a0-46326220e3ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:38Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac1bd016ad namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 682121b2-5176-432a-b415-cdf6f93605a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:46Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096ac8c6d9ac8 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 935c5a81-3153-484f-b947-67b441cc4d64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:57Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096b0a393361f namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: b5123426-7670-445f-8c42-1f51267e81b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 9afcb5e3-dc0c-4830-8dcc-63ffd2466e6d kind: Event lastTimestamp: "2026-07-09T09:58:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:57Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x.18c096b0a393b268 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 863e6af7-e099-4289-ae6e-e2d795a10de9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 67dc3fe5-bdd1-4d74-920c-5bc41d0a72fa kind: Event lastTimestamp: "2026-07-09T09:58:27Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-hz54x' metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:27Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c096a9a59aadd7 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 4a829c8c-679e-43e3-ad21-825fbb459c93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 288cdc0a-57d6-4661-96d3-073c3a8c56c1 kind: Event lastTimestamp: "2026-07-09T09:58:27Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:27Z" name: isvc-autogluon-ts-v1-predictor.18c096a99f56cfdd namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 6d1c97c3-0508-4b62-ad94-b921d78283de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:58:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: f11483dc-d91d-4a27-a1c5-4397cf304657 kind: Event lastTimestamp: "2026-07-09T09:58:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:58:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:57Z" name: isvc-autogluon-ts-v1-predictor.18c096ad1e84c906 namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 3ff09b41-7a37-4e9f-b9e4-2e84587e1b8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:58:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: f11483dc-d91d-4a27-a1c5-4397cf304657 kind: Event lastTimestamp: "2026-07-09T09:58:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:58:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:57Z" name: isvc-autogluon-ts-v1-predictor.18c096ad1ef0de6d namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 363db9e2-e674-411e-a3b7-c68964a8f3c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T09:58:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 804dba07-b22c-4da0-904a-d2dd82d7326d kind: Event lastTimestamp: "2026-07-09T09:58:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:56Z" name: isvc-autogluon-ts-v1.18c096a99f5c2d4c namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 38637191-5048-4f18-98fb-1c9db6bf64f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 804dba07-b22c-4da0-904a-d2dd82d7326d kind: Event lastTimestamp: "2026-07-09T09:58:27Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:27Z" name: isvc-autogluon-ts-v1.18c096a99fb0a8ae namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: aa60a244-62cb-4beb-8f72-c18670016c21 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 804dba07-b22c-4da0-904a-d2dd82d7326d kind: Event lastTimestamp: "2026-07-09T09:58:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:27Z" name: isvc-autogluon-ts-v1.18c096a99fb13479 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: dcd2cda4-d22c-4913-8faa-2c54eb35bef1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: 804dba07-b22c-4da0-904a-d2dd82d7326d kind: Event lastTimestamp: "2026-07-09T09:58:56Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-09T09:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:56Z" name: isvc-autogluon-ts-v1.18c096b04d0ed2c2 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 45485047-b0d3-46f6-974d-6b7881d40dfd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:49:16.218257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17536" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0962938219c6f namespace: kserve-ci-e2e-test resourceVersion: "17543" uid: caae887f-e87f-4bed-9829-007a4d627112 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09629519dec73 namespace: kserve-ci-e2e-test resourceVersion: "17575" uid: 8ac8da69-59eb-44b1-b4a9-974202804f73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09629533e883c namespace: kserve-ci-e2e-test resourceVersion: "17577" uid: 8abd4955-2409-4870-8e7c-1e00e73835aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09629560546a9 namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: d3bdec4f-1988-4bbc-b77a-d31d10b945c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09629569924ba namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: 5d34d7a0-f986-437e-a6a2-75055df8f963 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:49:37Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-09T09:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:37Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0962e3497ec6e namespace: kserve-ci-e2e-test resourceVersion: "17740" uid: 603c9bdb-678e-4272-a29e-b9b448845616 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m10.59s (2m10.59s including waiting). Image size: 8939170585 bytes.' metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964c9c63ee0d namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: e9129d8a-e3c7-4736-91d5-f246ebcb7471 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964ca2c01459 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 223c80fd-043e-45c7-a106-2429fefc9694 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964ca4105ad4 namespace: kserve-ci-e2e-test resourceVersion: "18650" uid: bc861295-4fef-46f3-b923-cbb9c7ab7c0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964ca41a741d namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 7d51bcb9-ae03-46ef-98ec-48b697f23e3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964caafe5b5a namespace: kserve-ci-e2e-test resourceVersion: "18653" uid: 62e016aa-75ef-4949-93c8-1492226bfcba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964cac30dd79 namespace: kserve-ci-e2e-test resourceVersion: "18654" uid: cb9f59df-e2fc-410f-b05a-51e4ae3f4d93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:51:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:51:55Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c0964cded314f1 namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: fbfd8b3a-a101-49b2-bc3d-899173018d9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:52:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09650d8a1a4f9 namespace: kserve-ci-e2e-test resourceVersion: "18807" uid: 287e68d1-6972-4e20-84cd-bb94119326e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:52:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:06Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09650d8a20a0a namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: 43a43403-8b4a-4ccc-9ce2-a39f4ff449e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 2ec26e09-e732-4cef-886c-971a44ca358d kind: Event lastTimestamp: "2026-07-09T09:52:10Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:10Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc.18c09651c2cb8bc2 namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: 6d27d0ca-f214-4b18-bddf-a6eefa7a87b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17526" uid: 4c85219d-2c45-43f5-bcd9-81054590dec1 kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-zqkhc' metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c0962937a71aef namespace: kserve-ci-e2e-test resourceVersion: "17537" uid: 70cb8b9f-71f3-4f40-bbd5-08400440c327 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: b0c008fd-6473-4a7b-b2ad-7832f091a2d0 kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1-predictor.18c0962934b02c97 namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 69605964-ec61-4c81-8526-826351c4b10b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17523" uid: 1cb81fa8-bd36-43e8-9cb6-efc29dce9f9a kind: Event lastTimestamp: "2026-07-09T09:52:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:52:01Z" name: isvc-autogluon-v1-predictor.18c0962cb3f33f4a namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: a81c45b5-d202-4d4a-a7be-832c09817968 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17523" uid: 1cb81fa8-bd36-43e8-9cb6-efc29dce9f9a kind: Event lastTimestamp: "2026-07-09T09:52:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:52:01Z" name: isvc-autogluon-v1-predictor.18c0962cb464210b namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: 8a568a07-1d5e-48ce-af37-d10b20da3a76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: 693e179a-34f5-40c6-8f07-3ab2db6147ca kind: Event lastTimestamp: "2026-07-09T09:52:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:52:05Z" name: isvc-autogluon-v1.18c0962934e3f3ff namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 7ae6907a-a390-4796-9771-7ced9c7a197c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: 693e179a-34f5-40c6-8f07-3ab2db6147ca kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1.18c09629355d760f namespace: kserve-ci-e2e-test resourceVersion: "17564" uid: 93e3d591-dd36-46f4-9cbf-f24c137e3938 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: 693e179a-34f5-40c6-8f07-3ab2db6147ca kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:49:16Z" name: isvc-autogluon-v1.18c09629355db86b namespace: kserve-ci-e2e-test resourceVersion: "17565" uid: a85159f3-09d2-42a7-b1d5-bcf0c59d31b1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: 693e179a-34f5-40c6-8f07-3ab2db6147ca kind: Event lastTimestamp: "2026-07-09T09:52:05Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-09T09:52:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:52:05Z" name: isvc-autogluon-v1.18c09650a17d4b66 namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: aebb4c46-647f-4160-ab5b-80a284561fa8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:56:42.476972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096911f3924bc namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 91375818-7117-46c3-9938-2a4accd8ef53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-noslash-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c0969129b1c911 namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: 831f9d92-78fe-49aa-b8bb-a096fd98604a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:43Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:56:43Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096915c6c6467 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: e5cb5513-b505-4fba-a415-553b67ba025a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:43Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096915dd7565d namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: dd537ed5-fbf6-4787-b770-e1154c3db624 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:43Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c0969160b1d978 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: 234ecb8a-d0e3-4338-975a-eea6905249ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:43Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c09691612238e2 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: c9174edf-8598-45a6-bcd7-84d98d8762f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c09692950358b5 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: db39fc84-801a-4790-af27-2d5e81b21bc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096929816b9c5 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: cdf4406e-bdcb-4984-b3b0-38413aa06e24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c0969298a512ef namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: c0b85a82-cd39-478e-814b-346c29f7172a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c0969298af0758 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: c384a850-b842-4995-99af-522fbb389ca3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096929be9ae94 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: 3b8f5f1c-c31d-41d3-a970-12e59e361747 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:56:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096929c5b93ab namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: e54f8566-2098-40fa-ad16-9158df5a82cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:58:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096a871e7f638 namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 5bd9b1ad-6ffa-48b5-befe-9be4896b3a4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 2fe7a350-91ef-44f9-8d7b-dda915d5b40d kind: Event lastTimestamp: "2026-07-09T09:58:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:58:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx.18c096a871e86aac namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 03f9266d-9369-4647-86bf-9d53732c51d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 4af09fd1-384f-4f1b-b700-ce49b047f236 kind: Event lastTimestamp: "2026-07-09T09:56:42Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-lm7jx' metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c096911eb83ea3 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: 9c8ca054-091a-412c-9472-e568bc7d942d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 8e5b3fa3-d7bf-428b-adf3-f99c89915ba5 kind: Event lastTimestamp: "2026-07-09T09:56:42Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash-predictor.18c0969119d40415 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 328af42a-c5fe-4a1e-bd84-3aadeb3e9de4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:56:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: e7fb6c42-fbba-4cee-be5b-b61bee5f64be kind: Event lastTimestamp: "2026-07-09T09:57:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:27Z" name: isvc-autogluon-v2-noslash-predictor.18c0969498e658ff namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: f3193978-41af-4763-a181-ca0dc7613b3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:56:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: e7fb6c42-fbba-4cee-be5b-b61bee5f64be kind: Event lastTimestamp: "2026-07-09T09:57:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:27Z" name: isvc-autogluon-v2-noslash-predictor.18c09694996242b0 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 1e466c84-2b7d-49fa-983e-5f79ab954202 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: e7fb6c42-fbba-4cee-be5b-b61bee5f64be kind: Event lastTimestamp: "2026-07-09T09:58:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:12Z" name: isvc-autogluon-v2-noslash-predictor.18c096981845e858 namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: c9aaeabe-e444-4e72-9094-cbf89e0e5ec3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:57:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: e7fb6c42-fbba-4cee-be5b-b61bee5f64be kind: Event lastTimestamp: "2026-07-09T09:58:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:58:12Z" name: isvc-autogluon-v2-noslash-predictor.18c0969818b412b0 namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 51944019-3b29-411a-8233-bcea89b10698 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 46d39a0f-f394-4541-96b1-22a01ac71a49 kind: Event lastTimestamp: "2026-07-09T09:58:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:21Z" name: isvc-autogluon-v2-noslash.18c0969119e336d1 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 72ecc7c9-db60-4d58-b627-5964c1293b7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 46d39a0f-f394-4541-96b1-22a01ac71a49 kind: Event lastTimestamp: "2026-07-09T09:56:42Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash.18c096911a6f7c5f namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 575aed46-0c29-4eaf-82a9-b8f9d65e9976 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 46d39a0f-f394-4541-96b1-22a01ac71a49 kind: Event lastTimestamp: "2026-07-09T09:56:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:42Z" name: isvc-autogluon-v2-noslash.18c096911a6fc2d1 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 7e88e5ac-2531-4ee2-b9f8-3fd5ffdf7c88 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 46d39a0f-f394-4541-96b1-22a01ac71a49 kind: Event lastTimestamp: "2026-07-09T09:58:21Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-09T09:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:58:21Z" name: isvc-autogluon-v2-noslash.18c096a82c80153b namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 72194bb1-98d2-423a-8a26-2d52c169942b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:52:11.769497Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-zn8lw to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:52:11Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965217ccf06a namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: c964c107-3ce2-470c-9901-d7737dfb7459 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:52:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:12Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965225993368 namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: 0d3b0281-29ef-4386-b199-0097adf9871f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18885" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:13Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:52:13Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c096528dbccd8a namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: bbb745cd-32f2-4e17-bf22-2b53b9457e79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:13Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c096528f25ad52 namespace: kserve-ci-e2e-test resourceVersion: "18911" uid: 16998165-fa71-4586-9540-7866129f3012 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:13Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965295b257a4 namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: abe5e659-93a0-4b2b-86a3-754a94366e2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:13Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965296ffa083 namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 4724f022-ef83-4e6d-b6d4-684903f9ed58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c09657659b076f namespace: kserve-ci-e2e-test resourceVersion: "19061" uid: d751fafe-ab53-4691-97bd-4cd7509c95b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c09657689a86d5 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 701b8066-738f-4ad6-b456-9b837dca957f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965768fb725c namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 558f4a6b-3711-4556-af51-de5ccbc2fe29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0965769057ede namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: 8ac92de6-c561-4325-a947-2cd6d17f40c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c096576c6655e2 namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: 534223ca-8204-4fc0-8cf5-b8956ebf9437 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:52:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:52:34Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c096576ce176e8 namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: a867f1b3-a562-4ac6-a0eb-db6f7057d2f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:54:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:21Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0967064f5478a namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: 98f72f9b-75be-4213-8e41-dfe24b10d581 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:54:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:21Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0967064f5cf15 namespace: kserve-ci-e2e-test resourceVersion: "19840" uid: 401ad47a-5ba8-438a-9a79-ca55d406541e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:54:22Z" message: 'Readiness probe failed: Get "http://10.134.0.38:8080/v2/models/isvc-autogluon-v2/ready": dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:22Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c0967080f56a59 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: d4b70b71-fab6-4d64-ab22-164289d79881 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-zn8lw namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 92d0cf50-e00a-4f3c-9d62-51aba1026713 kind: Event lastTimestamp: "2026-07-09T09:54:22Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:22Z" name: isvc-autogluon-v2-predictor-67b849d456-zn8lw.18c096708bd136b9 namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: a433d687-c3b7-4f3c-b56a-7d32435bfc16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 370ba9af-413c-4510-8307-3b890bf9c9d3 kind: Event lastTimestamp: "2026-07-09T09:52:11Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-zn8lw' metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:52:11Z" name: isvc-autogluon-v2-predictor-67b849d456.18c0965217725a8d namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: a842a5bd-160c-4156-9eae-d8108cf65bf2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:52:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 784c6184-ab1a-492a-82eb-464d0f62253d kind: Event lastTimestamp: "2026-07-09T09:52:11Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:52:11Z" name: isvc-autogluon-v2-predictor.18c0965212bad664 namespace: kserve-ci-e2e-test resourceVersion: "18864" uid: 2d4ed1d1-8908-45c6-9d24-4eb64decb723 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18862" uid: e9592442-c6d9-48ab-9266-e5144af0659b kind: Event lastTimestamp: "2026-07-09T09:53:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:53:11Z" name: isvc-autogluon-v2-predictor.18c0965591abdf93 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 6a15f885-ebd6-4288-9cba-8872037c1755 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18862" uid: e9592442-c6d9-48ab-9266-e5144af0659b kind: Event lastTimestamp: "2026-07-09T09:53:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:53:11Z" name: isvc-autogluon-v2-predictor.18c09655921ed0a6 namespace: kserve-ci-e2e-test resourceVersion: "19331" uid: 675ad2fb-2260-4209-a824-365cb0f1fd1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:52:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: e9592442-c6d9-48ab-9266-e5144af0659b kind: Event lastTimestamp: "2026-07-09T09:54:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:11Z" name: isvc-autogluon-v2-predictor.18c0965910e5d75f namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 798f2dce-f95f-4457-8371-34c08f864951 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:52:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: e9592442-c6d9-48ab-9266-e5144af0659b kind: Event lastTimestamp: "2026-07-09T09:54:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:11Z" name: isvc-autogluon-v2-predictor.18c0965911585390 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 12371a41-0510-4385-a1e7-b9e90c2d41e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:54:27.124387Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c096719b94aa3f namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: d838f590-0999-4fbd-8f8c-b808b1c71b94 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09671b4ff43ea namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: fa3dcefc-b4a6-4863-8426-9a25b565b253 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09671b68bfdab namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 3ad8ec2c-01d2-4f31-b47e-8653e4896935 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09671b96dae3d namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: ca0ffd80-c189-455d-b52f-182159600144 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09671b9e2f3d7 namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: 3b0c4293-f74d-49f4-9ff2-5d403fc8b90c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c096789d820923 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: 2fec07f0-99f7-44df-b11a-970073782a9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09678a06bab07 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: bcb8ea15-9f35-4741-a9dc-975ae3444a5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09678a0f67cd8 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 457d477f-ef49-493e-a4e8-c0c76d67b3af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09678a0ff8e59 namespace: kserve-ci-e2e-test resourceVersion: "20140" uid: 2ec92c96-f056-47b8-a8d7-6e28d899415b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09678a3d9428f namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 8c3c9e83-1713-40c8-9bb9-d2d877edb56d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:54:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:57Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c09678a439c2b4 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 4cc180ee-7904-4f1f-818e-d8016fedc7ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:56:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:37Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c0968fe7fe0c9d namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: b0e8361f-058d-4938-8516-3507461a7258 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:56:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:37Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c0968fe7fe9a06 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 9a0a9176-e2f4-4272-a2b5-146bc43cbef7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 2412a8d5-c29f-4761-be9a-c368c9316961 kind: Event lastTimestamp: "2026-07-09T09:56:37Z" message: 'Readiness probe failed: Get "http://10.134.0.39:8080/v2/models/isvc-autogluon-v2-variants/ready": dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:37Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c.18c0968fef82c4fb namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 758468b2-0129-4f7d-a379-0a917c6fee30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 8fc396cf-6c26-4276-8a26-46dbfbef298d kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-lxp9c' metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c096719b259c26 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: eaefa938-07f6-4382-b8b3-8504d61015c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 408c506a-f31b-4cec-ac34-d026dbd9aa53 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants-predictor.18c0967196726017 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 3d259f4f-9c89-40c0-a6ad-6fd5e5a3b33b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 93384b2e-23b7-4c5f-b652-826161549662 kind: Event lastTimestamp: "2026-07-09T09:55:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:27Z" name: isvc-autogluon-v2-variants-predictor.18c0967515b896ec namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 5ab45284-1e7a-4657-b5aa-c67acccd2881 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 93384b2e-23b7-4c5f-b652-826161549662 kind: Event lastTimestamp: "2026-07-09T09:55:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:27Z" name: isvc-autogluon-v2-variants-predictor.18c09675160af918 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 6359a465-671f-40e4-959d-51eec2913410 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 93384b2e-23b7-4c5f-b652-826161549662 kind: Event lastTimestamp: "2026-07-09T09:56:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:27Z" name: isvc-autogluon-v2-variants-predictor.18c0968312c82cb2 namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: e3eb597e-3fc5-427d-9a37-aeeb74b3eb25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 93384b2e-23b7-4c5f-b652-826161549662 kind: Event lastTimestamp: "2026-07-09T09:56:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:27Z" name: isvc-autogluon-v2-variants-predictor.18c09683133d7ead namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 5f49b948-453c-4f69-8d7b-95d219e36fcb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: d1b3462c-7ade-4a9e-a176-b242efe83222 kind: Event lastTimestamp: "2026-07-09T09:56:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:27Z" name: isvc-autogluon-v2-variants.18c09671966e508c namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 62b50a2d-8218-4488-b952-739dd7d282c9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: d1b3462c-7ade-4a9e-a176-b242efe83222 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants.18c0967197240055 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: cd29880c-638a-4ed9-8be3-b446a7b2d616 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: d1b3462c-7ade-4a9e-a176-b242efe83222 kind: Event lastTimestamp: "2026-07-09T09:54:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:27Z" name: isvc-autogluon-v2-variants.18c0967197244e10 namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: eae30667-c3ae-4c71-ac3d-83f7d52cdd8b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: d1b3462c-7ade-4a9e-a176-b242efe83222 kind: Event lastTimestamp: "2026-07-09T09:56:27Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-09T09:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:27Z" name: isvc-autogluon-v2-variants.18c0968da0d67c86 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 99885c92-737e-48d5-85cd-35e18ba7f411 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-09T09:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 346b33f3-443f-41af-bc1a-10662af2d90c kind: Event lastTimestamp: "2026-07-09T09:54:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:12Z" name: isvc-autogluon-v2.18c0965212caafcd namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 61e52115-dce7-4661-b2c1-ec51c75f91e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 346b33f3-443f-41af-bc1a-10662af2d90c kind: Event lastTimestamp: "2026-07-09T09:52:11Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:52:11Z" name: isvc-autogluon-v2.18c096521352ba08 namespace: kserve-ci-e2e-test resourceVersion: "18882" uid: 0986e48a-f781-4995-ac2d-2e38d32b45b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 346b33f3-443f-41af-bc1a-10662af2d90c kind: Event lastTimestamp: "2026-07-09T09:52:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:52:11Z" name: isvc-autogluon-v2.18c096521352f511 namespace: kserve-ci-e2e-test resourceVersion: "18884" uid: ef085665-2772-473e-bb0a-a6a84f139b28 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 346b33f3-443f-41af-bc1a-10662af2d90c kind: Event lastTimestamp: "2026-07-09T09:54:12Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-09T09:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:12Z" name: isvc-autogluon-v2.18c0966e32cefb1c namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 4791918f-2968-4867-8a90-5816c32ec643 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:47:51.031976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-7997fc58c6-r7cl4 to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c0961562a20b83 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 6727e8e3-5605-46ca-abc3-bef97b32d492 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16845" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:51Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096157c0886be namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: 619ac312-2dfe-42e9-b631-ad0dfb5ee62f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096157dad5c44 namespace: kserve-ci-e2e-test resourceVersion: "16855" uid: 1c7e1d52-1a71-4669-85e3-b336678c648a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096158042e245 namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: 821318b2-1b33-4e78-aba1-816be0cf00d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c0961580b0f645 namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: b726b45e-2aa0-42e6-a1ee-d3b2c430956d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169393e970 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 36ce9933-c8c9-4da2-a259-82f238e0ed2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c0961696c6616c namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 19b5f684-4e80-46fb-9b9a-50ca953fb9b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169736d88c namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 4e0f0747-9120-4c94-b060-629a5e554f5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c09616973f7163 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: bc0d96c6-50ad-416a-ba25-7e48eba47ee9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169a0eff6e namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: c646829a-f346-4292-b2ea-082e2ed57e09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169a7d2200 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: dae0a6e2-a4f9-4a1a-b624-7d90e663b908 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:b91ab915596b4f349832befac95e931936365ada90787fcb1c8ef485ef1e3a3a" already present on machine metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169a87d73e namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 8694499f-42cf-4fa5-81c6-c8671c65ddd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169dc0b269 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 71fb2a5c-61a3-4a37-ad15-2c2637e488e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:47:56Z" message: Started container agent metadata: creationTimestamp: "2026-07-09T09:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:56Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096169e2fe024 namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: eb7a1e0a-e01e-4216-bb0b-8f742bd556c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T09:47:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:27Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c09616cfa6c7fb namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: 4bfdd08d-3f08-4020-bf71-37ad30f0253c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:47:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:47:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:27Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096170b80eb09 namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: bcb74ba2-9210-4234-acd0-a8bd8e9295a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c0962934021f9f namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: a63e66eb-98fd-4046-99e6-82a19a9efdfb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c0962934025d75 namespace: kserve-ci-e2e-test resourceVersion: "17527" uid: cb3cce03-9d3d-410f-8428-abd5cab44547 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c096293402a376 namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 221d15ed-c2bc-4636-98c3-643a4c1abb8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-7997fc58c6-r7cl4 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 778f5ed3-0ef1-4f6a-81cd-4b22c84bbd4d kind: Event lastTimestamp: "2026-07-09T09:49:24Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:24Z" name: isvc-logger-predictor-7997fc58c6-r7cl4.18c09629e7bdc011 namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: d41b4715-8981-4f87-8106-259217de5034 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-7997fc58c6 namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: 1ae29ca8-51b5-434a-840b-68659ec72272 kind: Event lastTimestamp: "2026-07-09T09:47:51Z" message: 'Created pod: isvc-logger-predictor-7997fc58c6-r7cl4' metadata: creationTimestamp: "2026-07-09T09:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:51Z" name: isvc-logger-predictor-7997fc58c6.18c096156196e5ff namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: a898280c-0709-42fe-8b8f-19e89471e7a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: ab74defb-9183-46ee-9fc1-6f1f323a52d8 kind: Event lastTimestamp: "2026-07-09T09:47:50Z" message: Scaled up replica set isvc-logger-predictor-7997fc58c6 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:50Z" name: isvc-logger-predictor.18c09615599230ef namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: a421e0bf-a117-495d-ba91-c7a688d433ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 06905b8f-7be1-4770-841e-8726526a75b8 kind: Event lastTimestamp: "2026-07-09T09:48:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:20Z" name: isvc-logger-predictor.18c09618d821cb4f namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: f253749f-8b47-4bcc-9b97-a97b34a9213d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 06905b8f-7be1-4770-841e-8726526a75b8 kind: Event lastTimestamp: "2026-07-09T09:48:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:20Z" name: isvc-logger-predictor.18c09618d86e4920 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: f03f149a-5b0e-450c-99ee-5b712cb79bfe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 06905b8f-7be1-4770-841e-8726526a75b8 kind: Event lastTimestamp: "2026-07-09T09:49:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:49:05Z" name: isvc-logger-predictor.18c0961fd6ec993b namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: d083cae1-53db-425d-a0e0-570384d536e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 06905b8f-7be1-4770-841e-8726526a75b8 kind: Event lastTimestamp: "2026-07-09T09:49:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:49:05Z" name: isvc-logger-predictor.18c0961fd75ad5f0 namespace: kserve-ci-e2e-test resourceVersion: "17418" uid: dacd583f-77b7-4a9f-9646-64a98c9b99d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 95f1c859-94cc-4df5-9639-3a733a379df4 kind: Event lastTimestamp: "2026-07-09T09:49:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:49:07Z" name: isvc-logger.18c096155991fd82 namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: 4a6893e5-1c96-417c-a09e-eb2dd0c0d244 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 95f1c859-94cc-4df5-9639-3a733a379df4 kind: Event lastTimestamp: "2026-07-09T09:47:50Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:50Z" name: isvc-logger.18c096155a386595 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 36435172-0b73-4407-a95e-6c388172e7f5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: 95f1c859-94cc-4df5-9639-3a733a379df4 kind: Event lastTimestamp: "2026-07-09T09:49:07Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-09T09:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:49:07Z" name: isvc-logger.18c096272eb6bef7 namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: bea79dcb-1d25-4e2c-8feb-1a8c5bb4f4bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:45:55.828532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fa8ff8f2cb namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 9d90137c-a896-48f6-af01-cf4518cf04d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:45:56Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:45:56Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095faa9390e81 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 66fb3751-9b1b-47e3-be80-b099ab77c517 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:45:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:56Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095faaaf4a90c namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: d2dcf652-7842-40ac-bb57-685411467656 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:45:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:56Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095faadcc9b8b namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 3826f161-5114-4720-900d-d670a6c23bd2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:45:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:56Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095faae3aea32 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 53a1d595-801a-4006-aa1c-5aad4f92260e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbac1f5733 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: c8374277-1d5c-47d6-8bd8-6749ec7f4176 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbafb33cfb namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: def8390a-4877-417a-ab6d-e53dec2177ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb017dd70 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 892ad8b7-34c2-43b2-a5ef-544a3465392f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb0234f80 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: ec44fcdc-98ca-45ca-aa6f-ed88e1e814e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb2baa2a7 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: f7cc258a-edfa-4911-ae2b-a28f309d1c9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb31f89e6 namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 49fdb365-3ca7-4f47-9fd2-bce10fa51f8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:b91ab915596b4f349832befac95e931936365ada90787fcb1c8ef485ef1e3a3a" already present on machine metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb329e778 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: c9d345f9-8dbf-4554-b4fe-4f059d0f8545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb6cb472f namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: a5f9fcd0-8bd9-4333-9c9b-47835fd47c83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:46:00Z" message: Started container agent metadata: creationTimestamp: "2026-07-09T09:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:00Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbb72e4ac8 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 1b19778f-2a8f-46fb-a348-99110f93c2b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:38Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fbe855ed3c namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 12b26dee-0eaa-412d-96c0-5d8f043dd015 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T09:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:38Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c095fc24339aac namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 7eef0e4d-a6f3-4955-ac5f-f722b8731a49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:30Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c09610ad1f0577 namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: b0322817-3a31-49e3-8e5c-3109925052b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:30Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c09610ad1f2c37 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: df93c59c-6752-4646-81c4-5724337d8cb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:30Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c09610ad1fcd68 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 938bee5f-40b8-4de9-a1e5-b216cdd1ad02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 2c3f2b62-e10b-42fe-99d8-2163cb678e39 kind: Event lastTimestamp: "2026-07-09T09:47:48Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:48Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59.18c09611547e48d7 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: 1802c90b-48e8-4e7e-90f9-5522843748b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: a5d4464c-e6f6-4a7e-a6ef-cdd27698550d kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-7fd7b677bb-dsk59' metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-custom-predictor-7fd7b677bb.18c095fa8f528339 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 659cf292-add9-4562-a0f1-dda6803c3bfa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 1f87debb-1590-4768-b6bf-ae6fb5119569 kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-7fd7b677bb from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-custom-predictor.18c095fa86b98f9a namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 78031611-637d-44bb-b24e-012c53bb94f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 0d04dd92-6f9f-45fa-b06d-fdff381caf3e kind: Event lastTimestamp: "2026-07-09T09:46:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:40Z" name: isvc-sklearn-batcher-custom-predictor.18c095fe055bc679 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 97d11845-e0a2-49e7-838f-d7ea7d070d95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 0d04dd92-6f9f-45fa-b06d-fdff381caf3e kind: Event lastTimestamp: "2026-07-09T09:46:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:40Z" name: isvc-sklearn-batcher-custom-predictor.18c095fe05c71d7b namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: ecfcd314-cee5-490a-bfc8-3179be70715e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:46:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 0d04dd92-6f9f-45fa-b06d-fdff381caf3e kind: Event lastTimestamp: "2026-07-09T09:47:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:25Z" name: isvc-sklearn-batcher-custom-predictor.18c096018537aa7e namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 8611e3ba-e666-4808-a2d2-99f1e4507908 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:46:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 0d04dd92-6f9f-45fa-b06d-fdff381caf3e kind: Event lastTimestamp: "2026-07-09T09:47:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:25Z" name: isvc-sklearn-batcher-custom-predictor.18c0960185a52c8e namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: a7976ff5-7fa1-4d5c-ace3-45501a2476e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 6d07c26d-7696-4426-b698-764e98f03ef2 kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:18Z" name: isvc-sklearn-batcher-custom.18c095fa86b342db namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: d42171ab-ef0a-4724-a848-13590b53b0a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 6d07c26d-7696-4426-b698-764e98f03ef2 kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-custom.18c095fa873f8660 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: ab088303-cabd-497a-a695-56e5d8361c2a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 6d07c26d-7696-4426-b698-764e98f03ef2 kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-09T09:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:18Z" name: isvc-sklearn-batcher-custom.18c0960dde3b70f8 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: e4883fec-8557-4741-8184-f37a7859d054 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:44:10.503852Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e20a21f3c9 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: ea22599a-b815-4316-aed2-7ba0adb2acae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-batcher-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e21811eaad namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: cf2a44ac-eb42-49eb-a951-919c1d9474cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:11Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:44:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e247629b15 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: d5abe78c-4450-4f1d-bfa0-ea474a2bb247 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e24919ce3e namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: b510ab10-0e01-48e5-9ae4-2e2f94b19cca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e24bcf3607 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: a30212de-7168-4d6e-8d3d-5183bf61fe6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:11Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e24c32a059 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 4f236cb4-a24e-4bc2-86a0-015589df3226 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:16Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-09T09:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:16Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e3569a47ba namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 36a8dd33-9349-4feb-94b5-d8a1b5b4182f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:30Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 14.212s (14.212s including waiting). Image size: 1557794731 bytes.' metadata: creationTimestamp: "2026-07-09T09:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e6a5b7d2df namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 932e818b-9c94-4ea2-92f4-969b85ac66e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e6a8d83430 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: fac38686-62d2-494d-808e-64897d97f861 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e6a9351a6b namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 35562e63-90c2-473e-933d-12c0fe6b67ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:30Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:30Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e6a940ebe5 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: ef42f790-434e-4a70-b4e7-6e5790a82dd5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:32Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.269s (2.269s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:32Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e7307f8dda namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 82ded184-3f71-4f83-872a-6ed68b44a0bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:32Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e733d6399d namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 16d86ea9-78d4-47b9-8404-eda6c0dbb502 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:32Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e73433b314 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: a96262c1-2edb-4509-81e7-ce190b36f45f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:32Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:b91ab915596b4f349832befac95e931936365ada90787fcb1c8ef485ef1e3a3a" metadata: creationTimestamp: "2026-07-09T09:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:32Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e73440814c namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 2a85e25e-9d93-4bce-b229-a6625bc60d9a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:b91ab915596b4f349832befac95e931936365ada90787fcb1c8ef485ef1e3a3a" in 1.937s (1.937s including waiting). Image size: 239997027 bytes.' metadata: creationTimestamp: "2026-07-09T09:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:34Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e7a7bb9b4e namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 1394796d-0454-443f-bffc-125592c3bced reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-09T09:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:34Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e7ab14a13a namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: 192cad95-27c2-4acd-a965-ff123572ed09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:44:34Z" message: Started container agent metadata: creationTimestamp: "2026-07-09T09:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:34Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e7ab77237b namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: e4c8653e-5497-43b0-9116-d5fb704318b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:45:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:58Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e7ca3cdcc4 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 063eb4af-48a1-4e41-8a51-604cb267f70f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T09:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:45:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:58Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095e80622941a namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: e6d24e9a-46bc-4c54-a7a5-2eac983b9f66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095fa8e47d2c5 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 070ce95e-0f9f-485e-9370-0a988070356c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095fa8e4c5362 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: b86484be-38a5-4b64-ab2d-8ca60c104b31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095fa8e50ee6c namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: c5422e08-be83-4371-9f64-fdb797bf1204 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 6be1933a-7790-4515-9be3-86df8ae542f4 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:07Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46.18c095fae25f7ed6 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 876fc42b-3887-4797-8087-91c262afee4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-54dcb69b4c namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 7e1142f8-dfce-4874-b2e2-32cead260bbd kind: Event lastTimestamp: "2026-07-09T09:44:10Z" message: 'Created pod: isvc-sklearn-batcher-predictor-54dcb69b4c-mgf46' metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher-predictor-54dcb69b4c.18c095e209b47da1 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: e7d92b11-a178-460a-89f7-0988abb23a02 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: a6f0452d-9849-4582-9aef-d194e29c28e7 kind: Event lastTimestamp: "2026-07-09T09:44:10Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-54dcb69b4c from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher-predictor.18c095e200c18fba namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: f96fb6a4-fe7b-40c5-95c0-03490340d60d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 432dc44e-9939-4012-8c8e-9a4947ad865f kind: Event lastTimestamp: "2026-07-09T09:45:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:10Z" name: isvc-sklearn-batcher-predictor.18c095e586f3f434 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 9df5e12e-e366-4e9d-a3cd-6f565b2860fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 432dc44e-9939-4012-8c8e-9a4947ad865f kind: Event lastTimestamp: "2026-07-09T09:45:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:10Z" name: isvc-sklearn-batcher-predictor.18c095e5874cfa9e namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 161a3a71-8730-4230-907a-4a05ca62f16d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 432dc44e-9939-4012-8c8e-9a4947ad865f kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-predictor.18c095ec85ab99b5 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: aec59efe-555e-47e9-9083-439dbeacf5ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 432dc44e-9939-4012-8c8e-9a4947ad865f kind: Event lastTimestamp: "2026-07-09T09:45:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:55Z" name: isvc-sklearn-batcher-predictor.18c095ec861b033e namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 286daa79-e7d9-4bde-b0f9-131851361505 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 61a6c522-e765-449d-8064-cd0ae0527e3b kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-batcher.18c095e207bccf55 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 6ee6cad9-4cc7-4f47-bedf-b7029822a7df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 61a6c522-e765-449d-8064-cd0ae0527e3b kind: Event lastTimestamp: "2026-07-09T09:44:10Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher.18c095e208c9ac44 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: fc224b53-3108-4ad2-bd53-28e3735555d2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 61a6c522-e765-449d-8064-cd0ae0527e3b kind: Event lastTimestamp: "2026-07-09T09:44:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:10Z" name: isvc-sklearn-batcher.18c095e208ca09a6 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: f51c6d75-73e8-440f-82fa-c6537796e65e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 61a6c522-e765-449d-8064-cd0ae0527e3b kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-batcher.18c095f8d19bffd9 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: a261211e-0a26-4eb5-8bb7-0b8db4249882 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:47:30.898764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-jxrdf to ip-10-0-137-199.ec2.internal metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:47:30Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c09610b2999c5b namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 38488c12-1f17-4ecc-9dd3-09418824dcda reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b4d547db9-z5hcs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:31Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:31Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c09610c063e04f namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: bdf034ff-eafe-4efb-a4fa-22527cc4861c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:31Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:31Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:47:31Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c09610efc9f762 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 51604170-5251-4ef4-a816-ebf860d64406 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:31Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-09T09:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:31Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c09610f126376f namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: 7205cb97-9d12-46a2-b7ce-2bef0244694b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.356s (1.356s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c0961141fcb065 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 4d78eb2f-b25f-4c84-a92a-dde0a4f00977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c0961144f4987b namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 1fcb2a44-5b7c-4733-b9d2-d10e1cbdf991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c096114583f4c8 namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: b6ad94e2-a901-429b-8b4b-dd7d3be0b87f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c09611458df497 namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: ab7050b5-3dbc-450f-a8d1-319a064e0d4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c0961148a69f57 namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 18762c0f-4c39-4b0d-a018-41371b324dab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:47:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:47:33Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c096114913c7c7 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 57639c47-ac0d-46a1-8a2c-8ddfcbae936b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c096293b385e55 namespace: kserve-ci-e2e-test resourceVersion: "17560" uid: d5716213-296a-439f-88e1-538f2559e782 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-jxrdf namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 189b2206-4a46-4c59-bacb-320f1959a0a0 kind: Event lastTimestamp: "2026-07-09T09:49:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:49:16Z" name: message-dumper-predictor-6575bcfdfd-jxrdf.18c096293b38bc83 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 22f71a7b-272b-4385-868a-8c134ced105c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-199.ec2.internal source: component: kubelet host: ip-10-0-137-199.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: c37e4161-78dc-475e-908c-38e5e15de157 kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-jxrdf' metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:30Z" name: message-dumper-predictor-6575bcfdfd.18c09610b224b4cf namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 6093f201-7488-47fb-9cd9-9768fb4056b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: b11305a0-1d6e-4d52-a877-5e21ee664a9d kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:30Z" name: message-dumper-predictor.18c09610ae90ee05 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: d56cf2fc-6da5-4a0c-943b-6ce7849185ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 68995083-1824-4819-a57d-f14752b6d1ae kind: Event lastTimestamp: "2026-07-09T09:48:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:15Z" name: message-dumper-predictor.18c096142d48b0cc namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: 12fdb607-e11e-4b4a-86c7-95d1be1b7793 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 68995083-1824-4819-a57d-f14752b6d1ae kind: Event lastTimestamp: "2026-07-09T09:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:15Z" name: message-dumper-predictor.18c096142db73129 namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: c80bbbae-82a3-40ea-b932-cab4dc5ff74c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 68995083-1824-4819-a57d-f14752b6d1ae kind: Event lastTimestamp: "2026-07-09T09:48:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:00Z" name: message-dumper-predictor.18c09617acf09f94 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: 4daab997-5ae9-4989-b828-94b7915e0d9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 68995083-1824-4819-a57d-f14752b6d1ae kind: Event lastTimestamp: "2026-07-09T09:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:48:00Z" name: message-dumper-predictor.18c09617ad617c55 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 95fefe70-0a58-4679-a461-1c3006e2c77e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: a50f3fe7-4a47-4c32-b48e-db188b393952 kind: Event lastTimestamp: "2026-07-09T09:47:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:41Z" name: message-dumper.18c09610ae9d6289 namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: bab34261-1d5e-4dd8-be9a-a5acd874b612 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: a50f3fe7-4a47-4c32-b48e-db188b393952 kind: Event lastTimestamp: "2026-07-09T09:47:30Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:30Z" name: message-dumper.18c09610af08544b namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 544bbcec-c51d-4e05-8126-2925223c27f4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: a50f3fe7-4a47-4c32-b48e-db188b393952 kind: Event lastTimestamp: "2026-07-09T09:47:41Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-09T09:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:47:41Z" name: message-dumper.18c09613152f4d04 namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: c2e797df-0878-41ea-8a4c-0080dc6faac5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22269"