--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:32.544556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c0089f42576aac namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 09389355-85c0-4782-bcf7-6968e29d0f54 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:32Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c0089f5be618ce namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 7fda258e-9180-49d4-b3b5-e52940af304a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c0089f5dd1206b namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 2ad39dd4-2463-4f4f-9d3b-2136310b968f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c0089f60d751a9 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: d5f90bb2-755d-4a50-88b2-bf8e0323dd77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c0089f614b0b35 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: c826a8ba-daa2-4667-aed7-31709d557976 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1b2deab88 namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 6bdf614c-666a-4ba4-a447-810e0cb788b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1b6dc5a89 namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: 19337d34-0639-45f2-b776-e3cca226f3a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1b75617af namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 25782cd2-3898-4125-9c40-ce75759547c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1b75ff905 namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: 5d223302-b0cf-4a4e-84c8-36449da29ba3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1ba60d073 namespace: kserve-ci-e2e-test resourceVersion: "22120" uid: 5e159fbc-bbb6-4d87-a741-71eee3cc6ecd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:43Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a1baefd523 namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 034be89d-b1ce-489a-8a4a-32dc2e1e45bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:35:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:51Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a22ab268b0 namespace: kserve-ci-e2e-test resourceVersion: "22184" uid: 93ffc780-f09b-44f9-8f2b-668c0771799b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:36:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a6351c73e9 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c410dad3-e031-43f2-a5d7-37045fff003b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: bb68ae55-1110-490b-83a7-f60ecf41517c kind: Event lastTimestamp: "2026-07-07T14:36:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:02Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4.18c008a6351cec44 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: c47b170b-0d34-4dd1-9c25-d210c3f17a93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 7ac1f99b-d11f-4635-82ce-fe16625216af kind: Event lastTimestamp: "2026-07-07T14:35:32Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-ts6g4' metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c0089f41d663f9 namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 3d0a8d04-e192-4629-abb8-5896626395ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: b5f49910-62b7-4a99-a43c-4c9c591bc0b0 kind: Event lastTimestamp: "2026-07-07T14:35:32Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1-predictor.18c0089f2be19abb namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: b28c7f1a-c5c9-4070-ac2a-848fa415daf9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: daa864b4-e1ab-4107-a10c-d921d6946798 kind: Event lastTimestamp: "2026-07-07T14:36:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:02Z" name: isvc-autogluon-ts-v1-predictor.18c008a2aa1ba751 namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: 1d3e94c5-76e0-4288-b2d9-98f417b66342 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: daa864b4-e1ab-4107-a10c-d921d6946798 kind: Event lastTimestamp: "2026-07-07T14:36:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:02Z" name: isvc-autogluon-ts-v1-predictor.18c008a2aacbbda5 namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: 0df5d7b1-092b-4086-a91c-06256ae87c79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 0def7f7c-1b33-4c19-bd42-650e23625ef8 kind: Event lastTimestamp: "2026-07-07T14:36:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:01Z" name: isvc-autogluon-ts-v1.18c0089f2c5d436a namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: c7e067e5-be7e-42d8-8f00-89ab73bc679d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 0def7f7c-1b33-4c19-bd42-650e23625ef8 kind: Event lastTimestamp: "2026-07-07T14:35:32Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1.18c0089f2df043a9 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: dd815c95-0d9e-4c22-8f69-04ac7ebe5892 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 0def7f7c-1b33-4c19-bd42-650e23625ef8 kind: Event lastTimestamp: "2026-07-07T14:35:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:32Z" name: isvc-autogluon-ts-v1.18c0089f2df093f1 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 100abeff-dad4-43fd-8b03-c6fb05925c4d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: 0def7f7c-1b33-4c19-bd42-650e23625ef8 kind: Event lastTimestamp: "2026-07-07T14:36:01Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-07T14:36:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:01Z" name: isvc-autogluon-ts-v1.18c008a5ecb50dc6 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 074fed5a-0b29-498c-8d4d-927305bf7691 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:26:00.626981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:26:00Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0081a1965c899 namespace: kserve-ci-e2e-test resourceVersion: "17696" uid: e4958eeb-4db5-4bc4-baf1-735634bb105f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17712" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:26:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0081a33f1f726 namespace: kserve-ci-e2e-test resourceVersion: "17725" uid: a0ce847c-42e0-4e50-b6d0-5ad6f172dd41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0081a35f42125 namespace: kserve-ci-e2e-test resourceVersion: "17728" uid: 11baaa39-3ca4-4ca4-87ec-ba8d38431213 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0081a3866e7f9 namespace: kserve-ci-e2e-test resourceVersion: "17729" uid: c504e745-246a-46eb-b788-98598b1cbd85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:01Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0081a38f43a9f namespace: kserve-ci-e2e-test resourceVersion: "17730" uid: 33d657ab-d608-4f40-9fd1-1d0b69cc39a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:31Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0082154a73f3a namespace: kserve-ci-e2e-test resourceVersion: "17958" uid: 09e2d73b-3fe4-4f2d-87f5-1c62d2b9d4e3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m11.564s (2m11.564s including waiting). Image size: 8939170587 bytes.' metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0083ff67e29cb namespace: kserve-ci-e2e-test resourceVersion: "18871" uid: abaf9b7d-50d1-4620-b972-f75af5d2d500 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0083fffb6ae03 namespace: kserve-ci-e2e-test resourceVersion: "18872" uid: 713acff0-3ae4-4411-9bae-465ef88fe741 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c008400049c461 namespace: kserve-ci-e2e-test resourceVersion: "18873" uid: 2320e889-a562-4e66-9f85-98b85a48a117 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c008400055826e namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 2f04c9a1-90b9-4dec-af1f-89a3dd1ffc6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0084005b3169d namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: 34210c9b-8953-443e-a07b-e1a17e5698c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:28:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:43Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0084007185f41 namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: f6251316-4638-47e9-abfc-e988c446d679 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:29:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:11Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c008406fef5077 namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: 85c9276b-f4ce-48d3-82ef-90dee2125cdb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:29:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:10Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0084660a630f0 namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: 18361c46-b602-4f08-b5f5-ac9978794d8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:29:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:10Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c0084660a7484d namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 77ac6d0d-9e9a-4a54-8f13-4b9f8d75ff4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 8c443757-aebb-4999-9826-cdb6f82f27d4 kind: Event lastTimestamp: "2026-07-07T14:29:11Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:11Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn.18c008467dd554cf namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: bc80f874-c741-44ad-8366-0809c9f3786a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: 555f8fbb-2eef-4438-9fa6-e18af4fa6c86 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-2x7hn' metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:00Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c0081a18e9db5c namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: 988b94c1-8ddf-4f95-a4d7-e4fde37be31e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17662" uid: 7b91360b-2b92-4b15-88fa-5115e860eec6 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:00Z" name: isvc-autogluon-v1-predictor.18c0081a135bdd4e namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: f5ff4147-4a2b-4046-8290-7464aa827845 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:26:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 82eacf2d-5b0e-42dd-a719-8018a3b307c9 kind: Event lastTimestamp: "2026-07-07T14:29:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:00Z" name: isvc-autogluon-v1-predictor.18c0081d9224b9dd namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: d02bc284-b759-4a65-a28e-0e1451a0d6cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:26:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 82eacf2d-5b0e-42dd-a719-8018a3b307c9 kind: Event lastTimestamp: "2026-07-07T14:29:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:00Z" name: isvc-autogluon-v1-predictor.18c0081d92af7361 namespace: kserve-ci-e2e-test resourceVersion: "19004" uid: 9915544c-f9d9-4888-ad9f-944f3ae8ab1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 07e82a80-abe3-4b97-b086-055da42042f1 kind: Event lastTimestamp: "2026-07-07T14:29:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:29:01Z" name: isvc-autogluon-v1.18c0081a13527641 namespace: kserve-ci-e2e-test resourceVersion: "19021" uid: 0f1cd192-2dd6-46c7-a27d-b03ce3ea1adf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 07e82a80-abe3-4b97-b086-055da42042f1 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:00Z" name: isvc-autogluon-v1.18c0081a141837de namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 805d5205-5fc3-4bf2-9a08-4dc1beac982d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 07e82a80-abe3-4b97-b086-055da42042f1 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:00Z" name: isvc-autogluon-v1.18c0081a1418904b namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: e1cadd98-bcfe-4bd7-8eb6-17f5aa7f70f3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "19019" uid: 07e82a80-abe3-4b97-b086-055da42042f1 kind: Event lastTimestamp: "2026-07-07T14:29:01Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-07T14:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:29:01Z" name: isvc-autogluon-v1.18c00844317e54a5 namespace: kserve-ci-e2e-test resourceVersion: "19020" uid: 80ba9048-292f-40cc-8c8d-8d837dff94f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:33:17.058708Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0087fb6c17b44 namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 592d007d-1275-4552-a80a-43d9eb71c112 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0087fd1602fec namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: e97ea512-1a74-4965-b04c-d74d499ca7ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0087fd33d2a87 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 06a0d04b-1cfa-4ece-b51b-343c769fa763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0087fd71f26a1 namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 86a58868-d9e3-4695-bccf-e4816035c36d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0087fd7a13ee5 namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: f90d8a52-205c-4513-9528-2004d1d391d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c008870b5fab3e namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 775c103b-cbfb-4206-8026-cfc4bb6d6bc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c008870ec30cb3 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: c7c9d8de-d99c-4e06-965d-c052c5a4fbf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c008870f4429f6 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 097e202a-f756-486c-a758-213c6853a10c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c008870f502d7c namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 2e223910-c310-4e06-8e18-be5e3f07d56f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0088713f234e5 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: bb8aff46-1a5c-45ee-bed4-0372ae820417 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:33:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c00887147ea0d6 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: f19ee3e6-6c45-4b02-84c4-979429d5b65b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:35:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0089dfd2fd5fd namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: f3beb904-8d95-4f49-abc8-716219c390eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: b0cec7cf-687b-459b-8d9c-bf26569a1a24 kind: Event lastTimestamp: "2026-07-07T14:35:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:27Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc.18c0089dfd305cc8 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 4bc28d39-def8-4bd2-bb52-dafcee1a4222 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: a8fe2073-ee85-42fc-bca3-466cac9622ab kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-qtcnc' metadata: creationTimestamp: "2026-07-07T14:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:17Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c0087fb60b8aad namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 8a9f6bb1-90c8-490b-8887-057829d60ebf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 8c412fc6-8415-4c0e-8557-3307ea6910c0 kind: Event lastTimestamp: "2026-07-07T14:33:16Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:16Z" name: isvc-autogluon-v2-noslash-predictor.18c0087faa98bff4 namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: 079851b3-2c2b-4476-a29b-6161dbf47f39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 2311c22e-b27c-45ea-a438-da9307256ef2 kind: Event lastTimestamp: "2026-07-07T14:34:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:16Z" name: isvc-autogluon-v2-noslash-predictor.18c0088329f73816 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 1d1f7e75-4bad-470c-b47f-fede4c21de84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 2311c22e-b27c-45ea-a438-da9307256ef2 kind: Event lastTimestamp: "2026-07-07T14:34:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:16Z" name: isvc-autogluon-v2-noslash-predictor.18c008832a96964f namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 2daa30dd-62ed-4e2e-ab47-c0d231f3d9b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:34:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 2311c22e-b27c-45ea-a438-da9307256ef2 kind: Event lastTimestamp: "2026-07-07T14:35:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:17Z" name: isvc-autogluon-v2-noslash-predictor.18c0089128fa85a6 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 5e3b313b-c5e2-458f-ad1d-691eae729e55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:34:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 2311c22e-b27c-45ea-a438-da9307256ef2 kind: Event lastTimestamp: "2026-07-07T14:35:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:17Z" name: isvc-autogluon-v2-noslash-predictor.18c0089129819459 namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 44188c8e-8401-4b3e-8b37-b933ae715479 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 3ea40870-6c0d-44f7-8c64-6bcad0c4b434 kind: Event lastTimestamp: "2026-07-07T14:35:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:22Z" name: isvc-autogluon-v2-noslash.18c0087faa661dcf namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 9727852c-0f36-4612-8bec-ccf2f8795bd3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 3ea40870-6c0d-44f7-8c64-6bcad0c4b434 kind: Event lastTimestamp: "2026-07-07T14:33:16Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:16Z" name: isvc-autogluon-v2-noslash.18c0087faba71203 namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: 3ee46b57-0e67-4d8d-a1d4-5f47c868a0d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 3ea40870-6c0d-44f7-8c64-6bcad0c4b434 kind: Event lastTimestamp: "2026-07-07T14:33:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:16Z" name: isvc-autogluon-v2-noslash.18c0087faba75d24 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 7c3bd542-fe08-429d-aa53-703aeb5e1933 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: 3ea40870-6c0d-44f7-8c64-6bcad0c4b434 kind: Event lastTimestamp: "2026-07-07T14:35:22Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-07T14:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:22Z" name: isvc-autogluon-v2-noslash.18c0089cef859643 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 157ed947-cd46-49c8-a7c9-dfbd2d33d4c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:29:16.259755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19161" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-mm4w6 to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00847a605080e namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: d6c5c47f-9ca0-43ac-ade0-3899a68ac1a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19173" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00847c292226b namespace: kserve-ci-e2e-test resourceVersion: "19177" uid: 34467384-c112-4efc-8e93-10cdedc3ad11 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00847c7594ac0 namespace: kserve-ci-e2e-test resourceVersion: "19179" uid: ec56fafb-c978-40a5-a049-f32b563b5d2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00847cdc48518 namespace: kserve-ci-e2e-test resourceVersion: "19180" uid: 71d50161-ad47-46f2-8da3-745afb768305 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00847cf11a7a7 namespace: kserve-ci-e2e-test resourceVersion: "19181" uid: b2a3631f-4a34-4b5e-a03b-4dd0fe6b776c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084eb30391bd namespace: kserve-ci-e2e-test resourceVersion: "19397" uid: e247485e-302e-4e38-add3-336f161e0c82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084eb754e3b6 namespace: kserve-ci-e2e-test resourceVersion: "19399" uid: 7e807232-7642-41f8-ad1c-8ed2259d4a81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084eb7cd7a0d namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: d5c484d2-65c1-4aa0-a411-fbaa4cf611d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084eb7d9dfb0 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 4a22a73f-098e-4045-9e1d-34929d1a8a5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084ebbddf261 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 6c8a3100-89f7-4cc1-ac00-08f816691ecb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:29:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:29:46Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c0084ebc5e59bc namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 7c52f9f8-f82d-47c1-be4a-a9a9064d0568 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:31:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:26Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00865f1e8c35b namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 0e75c6dc-cb25-4d27-8df2-fab727edc502 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:31:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:26Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00865f1e987b7 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 7b21a884-0aaf-4827-89a5-afe774ba4479 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-mm4w6 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b2a7f32e-9f27-4859-bba4-3c2659fd1c40 kind: Event lastTimestamp: "2026-07-07T14:31:26Z" message: 'Readiness probe failed: Get "http://10.132.0.30:8080/v2/models/isvc-autogluon-v2/ready": dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:26Z" name: isvc-autogluon-v2-predictor-67b849d456-mm4w6.18c00865f5289e22 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 98b62f32-b90e-4908-9f3a-3cc0448aaf3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: 11c1f940-266f-4924-b7d6-5f2e4130aca4 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-mm4w6' metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor-67b849d456.18c00847a44bca62 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: 59c418f3-c190-435e-b004-91add68952b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: b530fee6-6098-4946-9a73-2f7478c955e6 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2-predictor.18c008479ab96e9f namespace: kserve-ci-e2e-test resourceVersion: "19148" uid: 1bc683fb-10a2-475f-946f-47555fc8b2ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:29:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: 19cd1fa2-53b7-4e43-9e6a-5987afdd84dd kind: Event lastTimestamp: "2026-07-07T14:30:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:16Z" name: isvc-autogluon-v2-predictor.18c0084b1be2bcea namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: 60a55873-5e59-4a27-80a9-8e811d894130 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:29:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: 19cd1fa2-53b7-4e43-9e6a-5987afdd84dd kind: Event lastTimestamp: "2026-07-07T14:30:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:16Z" name: isvc-autogluon-v2-predictor.18c0084b1c8323fa namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 3472af68-e138-4188-9125-5d3360e32fe6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 19cd1fa2-53b7-4e43-9e6a-5987afdd84dd kind: Event lastTimestamp: "2026-07-07T14:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:16Z" name: isvc-autogluon-v2-predictor.18c008591acf4b78 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 2d49ba4d-1675-4a71-92fc-544c4a91805a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 19cd1fa2-53b7-4e43-9e6a-5987afdd84dd kind: Event lastTimestamp: "2026-07-07T14:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:16Z" name: isvc-autogluon-v2-predictor.18c008591b66a7de namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: e81a4dbf-f938-427d-9d16-22c9707fe273 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:31:31.749774Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:31:31Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c0086731daa8cf namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 86d75a93-d487-49de-b7b6-e67a7bcad16f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:32Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:31:32Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c008674bb11724 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 5aa6aa6d-bd77-4772-88a2-e1539eed3eca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:32Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c008674ef852ee namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 4403a832-0ba2-43d1-81af-6a2ec368afbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:32Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c008675260b01b namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 3b6ae54a-5518-4a74-b0b1-79568f04143a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:32Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c0086752d02c8a namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 9759c2ef-b295-4e48-bb4b-c57868c176aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:37Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T14:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:37Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868a63c581e namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 9a7f2c45-9b2a-4063-b5dc-934a09265c57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:38Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868a91af32d namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 88972826-fe45-4866-8586-ab047c13781f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:38Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868a98f6eb8 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 2396d49a-9a27-4483-8ab2-b01ecdc51d29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:38Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868a99a4c7f namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: fc41c7bd-256a-4077-96d6-fd207cc59f38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:38Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868ad24f03a namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: a8ffd6d4-2e39-44d0-a819-a7d6519f7c79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:31:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:38Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c00868adcc1161 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 4ca1b7c8-94b9-4dd7-8332-e7f5049b9063 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:33:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c0087ea0a3446c namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 40f80322-6c7f-4337-95a3-bb385b0759cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 5996ae57-6222-4c5e-bdbb-a063d1e0d60a kind: Event lastTimestamp: "2026-07-07T14:33:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:12Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm.18c0087ea0a3c5e5 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 7e89d9c0-61dd-45fd-be5f-fbe549d0d28f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 6de51cec-cd13-4bba-92f6-286c6dde9d41 kind: Event lastTimestamp: "2026-07-07T14:31:31Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-zb8mm' metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:31Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c0086730f36d24 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 61aa34d8-c921-4cb0-9aaf-a5cdff9571e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: ced1dada-2df0-4975-8623-b104e3e6b96b kind: Event lastTimestamp: "2026-07-07T14:31:31Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:31Z" name: isvc-autogluon-v2-variants-predictor.18c008671d07bd1e namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 5b5faa34-2bb8-40ac-b503-cd6674e65fb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 9f0daa98-d57b-4099-a28a-4b72afbd0165 kind: Event lastTimestamp: "2026-07-07T14:32:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:16Z" name: isvc-autogluon-v2-variants-predictor.18c0086a9bf8ff5c namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 1cc99500-4fcd-4af1-b8f4-615ad1b2e4f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 9f0daa98-d57b-4099-a28a-4b72afbd0165 kind: Event lastTimestamp: "2026-07-07T14:32:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:16Z" name: isvc-autogluon-v2-variants-predictor.18c0086a9c5e5a75 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: bda6e85e-a14d-4235-894e-762ec5c7db97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:32:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 9f0daa98-d57b-4099-a28a-4b72afbd0165 kind: Event lastTimestamp: "2026-07-07T14:33:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:01Z" name: isvc-autogluon-v2-variants-predictor.18c0086e1bfb8629 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 661b1a6f-6772-4344-93e6-5450e8e66108 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:32:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 9f0daa98-d57b-4099-a28a-4b72afbd0165 kind: Event lastTimestamp: "2026-07-07T14:33:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:01Z" name: isvc-autogluon-v2-variants-predictor.18c0086e1caa0ec6 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: c6aac102-1f0a-4110-8e74-7383cbe6e90e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: ec3b4d6c-c0d7-4801-89b7-2f26352b00b4 kind: Event lastTimestamp: "2026-07-07T14:33:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:10Z" name: isvc-autogluon-v2-variants.18c008671d118d0c namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: f71a4e89-eb54-43ed-a46d-5540557a3e93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: ec3b4d6c-c0d7-4801-89b7-2f26352b00b4 kind: Event lastTimestamp: "2026-07-07T14:31:31Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:31:31Z" name: isvc-autogluon-v2-variants.18c008671ddd41f5 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: f83d42b0-3506-43e3-9a64-bd34a6da80cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: ec3b4d6c-c0d7-4801-89b7-2f26352b00b4 kind: Event lastTimestamp: "2026-07-07T14:31:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:31:31Z" name: isvc-autogluon-v2-variants.18c008671ddd95de namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: 126ada8f-7eff-4223-b6c3-817bb21550f0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: ec3b4d6c-c0d7-4801-89b7-2f26352b00b4 kind: Event lastTimestamp: "2026-07-07T14:33:10Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-07T14:33:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:10Z" name: isvc-autogluon-v2-variants.18c0087e35d647a9 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 0fe1c5fe-ba33-45a9-be99-9debd6a49dea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 48b4d418-3474-436b-b4dc-a97047c96044 kind: Event lastTimestamp: "2026-07-07T14:31:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:31:16Z" name: isvc-autogluon-v2.18c008479abf950f namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: dc7fe466-59fc-49ca-8242-0ad985e146f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 48b4d418-3474-436b-b4dc-a97047c96044 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2.18c008479b455dc4 namespace: kserve-ci-e2e-test resourceVersion: "19152" uid: de5f44c4-fbde-406f-beca-10e35b9edeae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 48b4d418-3474-436b-b4dc-a97047c96044 kind: Event lastTimestamp: "2026-07-07T14:29:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:29:16Z" name: isvc-autogluon-v2.18c008479b45b4b8 namespace: kserve-ci-e2e-test resourceVersion: "19153" uid: ce730435-8610-4150-93c4-c9cd40533936 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 48b4d418-3474-436b-b4dc-a97047c96044 kind: Event lastTimestamp: "2026-07-07T14:31:16Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-07T14:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:31:16Z" name: isvc-autogluon-v2.18c00863a9bf64dc namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 541efdf4-9506-4f66-bcfd-7e16197fb3d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:25.158354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-599b89d7c-s8n6c to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00803df0622be namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: d88fe10e-9e09-41fc-ba19-ce4891233b65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00803f9294fc7 namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: 7d9c3e83-ec44-437a-bcc9-d63e40214b70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00803fb87758b namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: e199dc83-9bec-4466-98ed-602a778c62cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00803fe53bd52 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 068804d5-8944-4fb3-8f61-65fb0aec877d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00803feebf72e namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: 0bce488c-5150-42cb-bb62-71803f11d8c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c00805079d1153 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: 740abcae-a046-4783-ae1b-07b45a4060ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050aae1502 namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: bef643ff-45c4-4110-8e50-828fae3f5403 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050b424719 namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: 8fd50111-a585-4218-927f-fbde2aa53c5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050b4e1329 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 84ca3eb0-75d8-4ab5-a40d-03c57965f903 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050e4ba1c4 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: cfe5c46e-a205-44e5-af16-f713dbe7bf3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050ec4a4a8 namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 373e0be7-4ace-4059-8785-25290f8b5f5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008050ed2982a namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: b6bf18cf-29df-4dee-81fc-c00d7ab286e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0080511a049c8 namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: be580c47-0c94-449b-818f-a304cbc18210 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0080512164532 namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: dd1d5b33-cf5a-4899-a1e3-b6c7918a0ef2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:08Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0080543aa5520 namespace: kserve-ci-e2e-test resourceVersion: "17787" uid: 237cc71a-2d38-45c0-9b49-981c1e18b728 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:24:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:24:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:08Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c008057f83f7b9 namespace: kserve-ci-e2e-test resourceVersion: "17788" uid: f501d8c1-f782-4ee6-9a04-ac76782b9d35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:00Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0081a1953fd5e namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: c1646ef4-d0ad-4611-9355-dd7eafa9b597 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:00Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0081a19545004 namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: 17880724-7659-46b0-8dde-9dd7a3097874 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:00Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0081a1954e7de namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 4a142aab-d7d5-49d4-bb66-0264afbddba0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:26:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-599b89d7c-s8n6c namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 94529fa6-8c8f-4312-a94d-5ea59319c9f1 kind: Event lastTimestamp: "2026-07-07T14:26:18Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:26:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:18Z" name: isvc-logger-predictor-599b89d7c-s8n6c.18c0081aafc702a4 namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: 955250fe-60b3-4387-883d-29a0fa546905 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-599b89d7c namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 99728640-44c7-471e-a1ac-4ac2c9c89188 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: 'Created pod: isvc-logger-predictor-599b89d7c-s8n6c' metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor-599b89d7c.18c00803de7d4702 namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 6b072cfb-af7d-4d8a-84ca-207fbf3f9bfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: c735257f-b3e2-44ac-91cb-46b5a6b3fdb4 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: Scaled up replica set isvc-logger-predictor-599b89d7c from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger-predictor.18c00803da85c8fb namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 8554574e-a9a2-403c-9051-0e9640767d29 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: fab2762f-9e0f-4db4-b9f9-4a5831fb8013 kind: Event lastTimestamp: "2026-07-07T14:24:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:55Z" name: isvc-logger-predictor.18c0080759417e0b namespace: kserve-ci-e2e-test resourceVersion: "17168" uid: d4a216a1-8fe2-4e1c-8c1c-d7b9cf57e74f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: fab2762f-9e0f-4db4-b9f9-4a5831fb8013 kind: Event lastTimestamp: "2026-07-07T14:24:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:55Z" name: isvc-logger-predictor.18c0080759bddfd2 namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: c889a8a1-919f-4d6b-8c01-8b264a848202 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: fab2762f-9e0f-4db4-b9f9-4a5831fb8013 kind: Event lastTimestamp: "2026-07-07T14:25:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:55Z" name: isvc-logger-predictor.18c0080e577f6f83 namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: be1e4b5f-b7ab-41ac-9de4-e607933b5db5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: fab2762f-9e0f-4db4-b9f9-4a5831fb8013 kind: Event lastTimestamp: "2026-07-07T14:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:55Z" name: isvc-logger-predictor.18c0080e57fbb5a0 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 01d3cfff-8daf-4dc4-8398-ec21c93bac7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 207da21a-c96e-41e6-bc33-c215a9c33a02 kind: Event lastTimestamp: "2026-07-07T14:25:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:48Z" name: isvc-logger.18c00803da9847a6 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 79802080-b46a-4544-bd1f-f14295d166b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 207da21a-c96e-41e6-bc33-c215a9c33a02 kind: Event lastTimestamp: "2026-07-07T14:24:25Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:25Z" name: isvc-logger.18c00803db32f632 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 480904c5-f9bc-4777-8594-ea5c8f442743 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17555" uid: 207da21a-c96e-41e6-bc33-c215a9c33a02 kind: Event lastTimestamp: "2026-07-07T14:25:48Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-07T14:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:48Z" name: isvc-logger.18c008173afb2127 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 6a750056-65d2-476c-a54b-0e6f2073fb93 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:22:29.982175Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007e90dfd2b35 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: db84445b-de63-4289-9557-de1c0f6a598f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:30Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:22:30Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007e927f27fad namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: c2f92621-330e-4155-996e-9ea43cfb4eca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:30Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007e92a688e49 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: bacc10ef-46b1-4ad9-8c06-e3a39670bdc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:30Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007e92d58950c namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 0e215a4a-b334-415a-a785-3d135b385c91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:30Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007e92dd48d96 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: f37962eb-21f5-480a-870d-0cbe3a02bc7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea236fa126 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 63bd89aa-baac-476c-8540-c8cf1d466ce5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea26a17936 namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: d8a52540-96f0-4222-a28a-c2b8d7fba39c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea27208117 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: c5a41ff9-60a5-433d-967f-f93bb8c97ab8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea272d687e namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: b961b947-c38c-4174-87df-f34b171331e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea2a533ba5 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: ba0c01d5-3a82-4fa4-904a-6d3d4cd14cbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea2ad4050d namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 59cd884a-6b21-4007-930f-9c4c99ee9a34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea2adf9677 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 8f55dafa-d4cb-4b71-a2bd-97dd468f26b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea2dd02f83 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 01123a62-5ac7-4550-a7bf-9cd6416e9f32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:34Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea2e4ee314 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 2bdb2ae2-e5c9-44b7-9f9e-3305588adcdc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:22:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:21Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea5fa7eb88 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 677162c6-d8df-4451-b3dc-d81a12c952af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:22:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:11Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ea5fb0d69b namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 6468ddab-0bb8-48e6-8bf5-d1dcb471858d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:05Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ff371c51e4 namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: ec51e7ad-89e4-4139-907b-787a4fe5809c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:05Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ff371ceaab namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 881ed77d-8988-4e39-8f13-c5a8f0951a2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:05Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ff371e4817 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 27115660-6a34-494c-ac20-4a632f453cbc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: cd41e939-70bc-44ca-ae94-c10fdb289557 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:21Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q.18c007ff8ffe7ca6 namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 96eab7ea-d5ab-4d0f-b1de-a3502a7db2cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: d2b1e8ff-ee14-4d8d-acca-b8fae4b68ec3 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-5fccccdbd5-q6w6q' metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-custom-predictor-5fccccdbd5.18c007e90cbb4ea0 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: cb924d30-191f-4419-bb38-af2a45c3977c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 20a08ba2-f524-487a-ab10-efebc7aacf06 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-5fccccdbd5 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-custom-predictor.18c007e903d2e305 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: 23b467e1-21c4-43e4-819a-73e0eda393d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 3512a273-a517-4778-84d1-c922d9f872d1 kind: Event lastTimestamp: "2026-07-07T14:22:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:59Z" name: isvc-sklearn-batcher-custom-predictor.18c007ec829a376b namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 00577620-9bf7-4d69-a095-b157640c0ace reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 3512a273-a517-4778-84d1-c922d9f872d1 kind: Event lastTimestamp: "2026-07-07T14:22:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:59Z" name: isvc-sklearn-batcher-custom-predictor.18c007ec8337b5e3 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f634b43c-c03a-4dd0-9b3d-9d4eb90557c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 3512a273-a517-4778-84d1-c922d9f872d1 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:59Z" name: isvc-sklearn-batcher-custom-predictor.18c007f38199bf6f namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 2bcefe88-bc8e-45b8-9b3a-0afd0763d908 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 3512a273-a517-4778-84d1-c922d9f872d1 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:59Z" name: isvc-sklearn-batcher-custom-predictor.18c007f38225b01f namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 9b5f3c05-2705-4dce-a4a1-8dc9d412dc45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: e74b651d-92e4-4e83-a196-5d48ba3bd344 kind: Event lastTimestamp: "2026-07-07T14:23:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:51Z" name: isvc-sklearn-batcher-custom.18c007e904479c0d namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 4c0558ca-e801-4dff-90fc-5db56c7cd48c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: e74b651d-92e4-4e83-a196-5d48ba3bd344 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-custom.18c007e904f4d7c9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 6baa0ee5-33f3-44d5-a244-29ffcb7f8ae2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: e74b651d-92e4-4e83-a196-5d48ba3bd344 kind: Event lastTimestamp: "2026-07-07T14:23:51Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-07T14:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:51Z" name: isvc-sklearn-batcher-custom.18c007fc18dad8af namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: bbd7803c-ad67-4dd2-81e9-bf8015f1d641 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:20:34.585194Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:20:34Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007ce2fcaf712 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 09e33c48-f411-46da-8504-639bdf6a43dd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:35Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:20:35Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007ce49bce350 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 5b52f30f-7a9c-4c6b-b1ee-701837e679ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:35Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" metadata: creationTimestamp: "2026-07-07T14:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:35Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007ce4b7e19c6 namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 342c3c33-faf7-42e7-a64d-facbd650d58b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:38Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" in 3.465s (3.465s including waiting). Image size: 301879255 bytes.' metadata: creationTimestamp: "2026-07-07T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:38Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007cf1a0c2e34 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 68280141-74d7-4abf-99cf-b7b97ab7dc7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:38Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007cf1e156f40 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: c24e43fe-f962-485d-9e2e-ff4e59f87af3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:38Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007cf1e9098a4 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: ac3f6cda-1368-4110-bdc8-566564ee7fac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:43Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:43Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d02ebf54c7 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: a3b9ef66-be28-4c6a-bcd5-7d7ac5cde8d1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.715s (15.715s including waiting). Image size: 1555845045 bytes.' metadata: creationTimestamp: "2026-07-07T14:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d3d772b2a4 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 8a21c32d-46ee-4117-955a-30c4af0ad05e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d3daf9ce01 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: ec3aa8d0-94b2-4619-8d39-cb78aeadea52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d3db7cefe6 namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 25367265-7995-435b-b536-3f3b9f224c57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d3db8c12e7 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 700f96f8-6eea-4a61-8955-59fd5b2ebdbd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:01Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.804s (2.804s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:01Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d482bc61b0 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: de038363-423f-4af1-888c-5d908753eee5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:01Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d485e4e96f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: c1d24599-ae0b-4cf9-94b4-2ab4dcc237d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:01Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d48661271b namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: b00c9543-1144-4b55-9f3f-3c0a4ecdccb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:01Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" metadata: creationTimestamp: "2026-07-07T14:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:01Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d4866e75b7 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 66835591-359f-4750-9783-b0399fd13215 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" in 2.614s (2.614s including waiting). Image size: 239997051 bytes.' metadata: creationTimestamp: "2026-07-07T14:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:04Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d5223e4882 namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: 9ae3ef3b-57c8-435a-9d29-dba04697ba7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:04Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T14:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:04Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d525d3c711 namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: 84074b90-c538-4c45-ad3e-61888148f40f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:21:04Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T14:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:21:04Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d52653cee9 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: dc7b73dd-4ef0-4b1f-871a-14ebda2b9afc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:21:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:38Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d5549cd5bb namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 14d218f3-445b-4c6f-a4d7-2699ac7cb882 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:38Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007d59081092a namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 29114a9a-126b-40bd-a8ba-f36588668795 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007e90d59549f namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 6ae5e37a-c241-4a3c-b2ad-133f9deb310e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T14:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007e90d59f06a namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 27474b1e-23c3-402d-bbde-68e9a9ee24d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:29Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007e90d5a9cf0 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: b068207e-b022-489f-a548-952e1c9d0db3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: f3b2aa00-ae0c-4b62-a95f-743a97da5d78 kind: Event lastTimestamp: "2026-07-07T14:22:46Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:46Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n.18c007e95adb5313 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 35e2ce82-ea77-4885-ba35-0dda65455b3c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-869f8b8c7d namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: a2ea6939-a644-426c-a973-f99326e5f877 kind: Event lastTimestamp: "2026-07-07T14:20:34Z" message: 'Created pod: isvc-sklearn-batcher-predictor-869f8b8c7d-92g7n' metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:34Z" name: isvc-sklearn-batcher-predictor-869f8b8c7d.18c007ce2f5bb213 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 54c441ce-44a9-46c6-8ff5-3f88d1fa8bf0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: a8fa46dd-cd94-4ff7-9746-d19401c77683 kind: Event lastTimestamp: "2026-07-07T14:20:34Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-869f8b8c7d from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:34Z" name: isvc-sklearn-batcher-predictor.18c007ce26fedefc namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 57e9170f-15a3-40cb-9616-75ce76d75762 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:20:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 096d32ba-4741-42e5-90e5-d03f8913f68c kind: Event lastTimestamp: "2026-07-07T14:21:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:21:34Z" name: isvc-sklearn-batcher-predictor.18c007d1acb26068 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 7368bb20-d7c1-4965-bd08-56bb5d775abe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:20:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 096d32ba-4741-42e5-90e5-d03f8913f68c kind: Event lastTimestamp: "2026-07-07T14:21:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:21:34Z" name: isvc-sklearn-batcher-predictor.18c007d1ad35077f namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 01931a28-e69a-48ca-bcaa-c400abb56986 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:21:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 096d32ba-4741-42e5-90e5-d03f8913f68c kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-batcher-predictor.18c007dfac20b419 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: a2beefe9-96f4-46ff-8541-ac5beec3612c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:21:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 096d32ba-4741-42e5-90e5-d03f8913f68c kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-batcher-predictor.18c007dfac9a0de6 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: a2007d3b-7459-48ef-9404-7b95a77ea313 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:20:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 1d12a7dc-c36e-49ab-beac-0a38bfe7c5ec kind: Event lastTimestamp: "2026-07-07T14:22:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:18Z" name: isvc-sklearn-batcher.18c007ce2d16397a namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 081d20c6-e343-4d5e-b030-f2717c032d08 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 1d12a7dc-c36e-49ab-beac-0a38bfe7c5ec kind: Event lastTimestamp: "2026-07-07T14:20:34Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:20:34Z" name: isvc-sklearn-batcher.18c007ce2d7c3cd9 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 350cb114-c933-4530-81c2-e80c1785026f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 1d12a7dc-c36e-49ab-beac-0a38bfe7c5ec kind: Event lastTimestamp: "2026-07-07T14:20:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:20:34Z" name: isvc-sklearn-batcher.18c007ce2d7c8e39 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 6b70fe15-ff0a-4304-97de-c32e54121a7c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 1d12a7dc-c36e-49ab-beac-0a38bfe7c5ec kind: Event lastTimestamp: "2026-07-07T14:22:18Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-07T14:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:18Z" name: isvc-sklearn-batcher.18c007e65f06d376 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 71ba9b5c-befe-44d2-b3af-848c15d4a9b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:05.123729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-xx8rd to ip-10-0-135-29.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ff34de0adc namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 8053aae6-1bcd-4f76-95de-262fe0b1b5f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bd4d68c6f-264fp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ff50c380e3 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: bcc75a8f-0f77-40e1-8aa1-33e449878f07 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ff524dcdc0 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 2e988b43-51b8-4e7a-9925-e69daf489f29 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:06Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.288s (1.288s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-07T14:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:06Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ff9f1f1d9f namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 78c140c2-0f8c-433f-afd3-dd00903c9375 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:06Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ffa301c02d namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: 8381ea2b-8a5c-45cf-9e8a-d42aadd924a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:06Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ffa37f9c2c namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 4a32690d-5f7a-4b09-9495-61e623af2a40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:06Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ffa38bdd37 namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: 8d4d2995-9e4a-4cda-99f3-ec8aa60f933b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:07Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ffa74a30f1 namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: e2cf005c-fbdd-4824-bd4b-2737e789de50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:24:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:07Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c007ffa83fd587 namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: ee812bd0-9482-4c8a-8286-76f754081588 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:00Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c0081a1d48a7c7 namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: c150efac-feef-42c1-a1f5-75d10b2a0180 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-xx8rd namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: e89ebb4e-4eae-4d6c-8da0-b099b36d3b73 kind: Event lastTimestamp: "2026-07-07T14:26:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:00Z" name: message-dumper-predictor-6575bcfdfd-xx8rd.18c0081a1d4918b1 namespace: kserve-ci-e2e-test resourceVersion: "17718" uid: 6f819f7e-83d8-4cdd-95fa-5f63671034ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-29.ec2.internal source: component: kubelet host: ip-10-0-135-29.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: ba9a2d70-34e3-47b9-95b6-4316eb4aa1a6 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-xx8rd' metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper-predictor-6575bcfdfd.18c007ff34613df1 namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 1d994d2e-e56a-4dd8-8c6b-d5c7d6136d7d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 7565ae29-da0d-48b6-81c6-ebd93fe1221f kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper-predictor.18c007ff2f1b7fa3 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 554f2fe8-86ba-4115-a490-8ee1704e46af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: ade3b800-f4fe-4425-9776-ab4e6403042e kind: Event lastTimestamp: "2026-07-07T14:24:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:35Z" name: message-dumper-predictor.18c00802b0480bb1 namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 9e048b6c-19d9-48cb-871e-20edcd7ccc61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: ade3b800-f4fe-4425-9776-ab4e6403042e kind: Event lastTimestamp: "2026-07-07T14:24:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:35Z" name: message-dumper-predictor.18c00802b0eaae9b namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 819ca3ef-c12d-4c6c-9c15-9ac843e9f14b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: a0b2626a-6263-4a61-b5e0-fafb80dac1a9 kind: Event lastTimestamp: "2026-07-07T14:24:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:15Z" name: message-dumper.18c007ff2f24092f namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: d9efaffc-4c9c-43d8-b056-68d5da62dd03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: a0b2626a-6263-4a61-b5e0-fafb80dac1a9 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:05Z" name: message-dumper.18c007ff2fb83361 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: f75e5f12-f648-4479-86e0-ec8ebb415535 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: a0b2626a-6263-4a61-b5e0-fafb80dac1a9 kind: Event lastTimestamp: "2026-07-07T14:24:15Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-07T14:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:15Z" name: message-dumper.18c008018d2235c7 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 0199c156-e4fe-42aa-8c45-73f38e0dee17 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22535"